Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<load_from_csv>
print('hi' )
test = pd.read_csv('.. /input/test.csv')
 train = pd.read_csv('.. /input/train.csv' )<count_unique_values>
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE=f'{path}glove6b50d/glove.6B.50d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'
train.select_dtypes(np.int64 ).nunique().value_counts().sort_index().plot.bar(color = 'blue')
 plt.xlabel('Number of Unique Values')
 plt.ylabel('Count')
 plt.title('Count of Unique Values in Columns of Type Int64' )<concatenate>
embed_size = 50
max_features = 20000
maxlen = 100
test['Target'] = np.nan
data = train.append(test, ignore_index = False)
 heads = data.loc[data['parentesco1'] == 1, :]<feature_engineering>
train = pd.read_csv(TRAIN_DATA_FILE)
 test = pd.read_csv(TEST_DATA_FILE)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
list_sentences_test = test["comment_text"].fillna("_na_" ).values
consistent = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 inconsistent = consistent[consistent != True]
for household in inconsistent.index:
actual = int(train[(train['idhogar'] == household)&(train['parentesco1'] == 1)]['Target'])
 train.loc[train['idhogar'] == household, 'Target'] = actual
consistent = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 inconsistent = consistent[consistent != True]
print('There are {} households with inconsistent Target labels.'.format(len(inconsistent)) )<groupby>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )
heads.groupby('v18q')['v18q1'].apply(lambda x: x.isnull().sum())
<data_type_conversions>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split())for o in open(EMBEDDING_FILE))
data = data.fillna({'v18q1':0} )<create_dataframe>
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
house_vars = ['tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5']
nulls = train.loc[train.v2a1.isnull() , house_vars].sum()
notnulls = train.loc[train.v2a1.notnull() , house_vars].sum()
pd.DataFrame(data = {'isnull': nulls, 'notnull': notnulls} )<feature_engineering>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch, score))
data.loc[data.tipovivi1 == 1, 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()<feature_engineering>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=True )(inp)
 x = Bidirectional(LSTM(50, return_sequences=True,dropout=0.1, recurrent_dropout=0.1))(x)
 x = GlobalMaxPool1D()(x)
 x = BatchNormalization()(x)
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 def loss(y_true, y_pred):
return K.binary_crossentropy(y_true, y_pred)
 model.compile(loss=loss, optimizer='nadam', metrics=['accuracy'] )
data.loc[(( data['age'] < 7)|(data['age'] > 19)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()<filter>
def schedule(ind):
a = [0.002,0.003, 0.000]
return a[ind]
lr = callbacks.LearningRateScheduler(schedule)
 [X_train, X_val, y_train, y_val] = train_test_split(X_t, y, train_size=0.95)
 ra_val = RocAucEvaluation(validation_data=(X_val, y_val), interval=1)
 model.fit(X_train, y_train, batch_size=64, epochs=3, validation_data=(X_val, y_val), callbacks=[lr, ra_val])
<define_variables><EOS> 
y_test = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv')
 sample_submission[list_classes] = y_test
sample_submission.to_csv('LSTM-submission.csv', index=False )