Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<predict_on_test>
np.random.seed(42)
 warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'
eng_stopwords = set(stopwords.words("english"))
ans = model.predict(test_data)
 subs['Target'] = ans
print(subs)
 subs.to_csv('sample_submission.csv',index=False)
<set_options>
train = pd.read_csv(".. /input/processing-helps-boosting-about-0-0005-on-lb/train_processed.csv")
 test = pd.read_csv(".. /input/processing-helps-boosting-about-0-0005-on-lb/test_processed.csv")
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 18
plt.rcParams['patch.edgecolor'] = 'k'<load_from_csv>
special_character_removal = re.compile(r'[^A-Za-z\.\-\?\!\,\
def clean_text(x):
x_ascii = unidecode(x)
 x_clean = special_character_removal.sub('',x_ascii)
 return x_clean
train['clean_text'] = train['comment_text'].apply(lambda x: clean_text(str(x)))
 test['clean_text'] = test['comment_text'].apply(lambda x: clean_text(str(x)))
 X_train = train['clean_text'].fillna("something" ).values
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test['clean_text'].fillna("something" ).values
pd.options.display.max_columns = 150
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<import_modules>
def add_features(df):
df['comment_text'] = df['comment_text'].apply(lambda x:str(x))
df['total_length'] = df['comment_text'].apply(len)
 df['capitals'] = df['comment_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.comment_text.str.count('\S+')
 df['num_unique_words'] = df['comment_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
train = add_features(train)
 test = add_features(test)
 features = train[['caps_vs_length', 'words_vs_unique']].fillna(0)
 test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features )
from collections import OrderedDict<data_type_conversions>
max_features = 10000
maxlen = 50
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train_sequence = tokenizer.texts_to_sequences(X_train)
 X_test_sequence = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train_sequence, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test_sequence, maxlen=maxlen)
 print(len(tokenizer.word_index))
map={'yes':1,"no":0}
for df in [train,test]:
df['dependency']=df['dependency'].replace(map ).astype(np.float64)
 df['edjefe']=df['edjefe'].replace(map ).astype(np.float64)
 df['edjefa']=df['edjefa'].replace(map ).astype(np.float64)
 train[['dependency','edjefe','edjefa']].describe()<concatenate>
spell_model = gensim.models.KeyedVectors.load_word2vec_format(EMBEDDING_FILE_FASTTEXT )
test['Target']=np.nan
data=train.append(test,ignore_index=True )<prepare_x_and_y>
words = spell_model.index2word
w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or known(edits2(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i,letter in enumerate(word)if i == 0 or letter != word[i-1]] )
X = [[0],[1],[2]]
Y = [ 1, 2, 1 ]
dtc = DTC(max_depth=1 )<train_model>
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words,501))
something_tw = embeddings_index_tw.get("something")
 something_ft = embeddings_index_ft.get("something")
 something = np.zeros(( 501,))
something[:300,] = something_ft
something[300:500,] = something_tw
something[500,] = 0
def all_caps(word):
return len(word)> 1 and word.isupper()
def embed_word(embedding_matrix,i,word):
embedding_vector_ft = embeddings_index_ft.get(word)
 if embedding_vector_ft is not None:
if all_caps(word):
last_value = np.array([1])
 else:
last_value = np.array([0])
 embedding_matrix[i,:300] = embedding_vector_ft
embedding_matrix[i,500] = last_value
embedding_vector_tw = embeddings_index_tw.get(word)
 if embedding_vector_tw is not None:
embedding_matrix[i,300:500] = embedding_vector_tw
for word, i in word_index.items() :
if i >= max_features: continue
if embeddings_index_ft.get(word)is not None:
embed_word(embedding_matrix,i,word)
 else:
if len(word)> 0:
embedding_matrix[i] = something
else:
word2 = correction(word)
 if embeddings_index_ft.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 else:
word2 = correction(singlify(word))
if embeddings_index_ft.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 else:
embedding_matrix[i] = something
dtc.fit(X,Y)
 print(dtc.tree_.threshold)
 print(dtc.tree_.impurity)
<define_search_space>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
self.max_score = 0
self.not_better_count = 0
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=1)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))
if(score > self.max_score):
print("*** New High Score(previous: %.6f)
 " % self.max_score)
 model.save_weights("best_weights.h5")
 self.max_score=score
self.not_better_count = 0
else:
self.not_better_count += 1
if self.not_better_count > 3:
print("Epoch %05d: early stopping, high score = %.6f" %(epoch,self.max_score))
self.model.stop_training = True
dtc.fit(X,Y,sample_weight=[1,2,3])
 print(dtc.tree_.threshold)
 print(dtc.tree_.impurity)
<count_values>
def get_model(features,clipvalue=1.,num_filters=40,dropout=0.5,embed_size=501):
features_input = Input(shape=(features.shape[1],))
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x, x_h, x_c = Bidirectional(GRU(num_filters, return_sequences=True, return_state = True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, x_h, max_pool,features_input])
 outp = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=[inp,features_input], outputs=outp)
 adam = optimizers.adam(clipvalue=clipvalue)
 model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
 return model
<define_variables><EOS> 
model = get_model(features)
 batch_size = 32
epochs = 1
gc.collect()
K.clear_session()
num_folds = 2
predict = np.zeros(( test.shape[0],6))
kf = KFold(n_splits=num_folds, shuffle=True, random_state=239)
 for train_index, test_index in kf.split(x_train):
kfold_y_train,kfold_y_test = y_train[train_index], y_train[test_index]
kfold_X_train = x_train[train_index]
kfold_X_features = features[train_index]
kfold_X_valid = x_train[test_index]
kfold_X_valid_features = features[test_index]
gc.collect()
K.clear_session()
model = get_model(features)
 ra_val = RocAucEvaluation(validation_data=([kfold_X_valid,kfold_X_valid_features], kfold_y_test), interval = 1)
 model.fit([kfold_X_train,kfold_X_features], kfold_y_train, batch_size=batch_size, epochs=epochs, verbose=1,
callbacks = [ra_val])
 gc.collect()
model.load_weights("best_weights.h5")
 predict += model.predict([x_test,test_features], batch_size=batch_size,verbose=1)/ num_folds
print("Done")
 sample_submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv")
 class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
sample_submission[class_names] = predict
sample_submission.to_csv('model_9872_baseline_submission.csv',index=False)