Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<compute_test_metric>
import sys, os, re, csv, codecs, numpy as np, pandas as pd
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
print("lgbm score:", np.mean(np.abs(y_test - lgbm.predict(X_test))))<predict_on_test>
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE=f'{path}fasttext-crawl-300d-2m/crawl-300d-2M.vec'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'
def predict(models, x, weights = None):
ans = []
y = 0
for i, model in enumerate(models):
if weights is None:
y += model.predict(x)*(1/len(models))
else:
y += model.predict(x)*weights[i]
return y
models = [cat_models, svm, rf, lgbm]<load_from_csv>
embed_size = 300
max_features = 20000
maxlen = 100
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 index = list(submission.index )<load_from_csv>
train = pd.read_csv(TRAIN_DATA_FILE)
 test = pd.read_csv(TEST_DATA_FILE)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
list_sentences_test = test["comment_text"].fillna("_na_" ).values
test = []
for seg_id in index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 test.append(gen_feature(seg['acoustic_data']))
test = np.array(test)
 test = scaler1.transform(test )<predict_on_test>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )
y_pre1 = predict(models, test, [0.3,0.3,0.2,0.2] )<split>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split())for o in open(EMBEDDING_FILE))
try:
del X_train, y_train, X_test, y_test
except:
pass
X_train, y_train, X_test, y_test = gen_data(feature_fun = rolling_feature,size=2000,test_size=200 )<choose_model_class>
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
class MyLSTM:
def __init__(self, data, labels, valid_data, valid_labels,input_shape = 149):
self.nn = self.bulid_lstm(data.shape[1], data.shape[2])
 cb = EarlyStopping(monitor='val_loss', patience=1, verbose=0, mode='min')
 self.nn.compile(optimizer="adam", loss="MAE")
 self.input_shape =(data.shape[1], data.shape[2])
 self.data = data
self.labels = labels
self.valid =(valid_data,valid_labels)
 def bulid_lstm(self, input_shape, feature_shape):
seq = keras.Input(shape=(input_shape, feature_shape))
x = CuDNNGRU(48 )(seq)
 x = Dense(10, activation='relu' )(x)
 x = Dense(units=1 )(x)
 model = Model(inputs=seq, outputs=x)
 model.summary()
return model
def train_on_batch(self, epochs = 50, batch_size=20, intervals=10):
losses = []
test_losses = []
for epoch in range(epochs):
for batch_i, data in enumerate(load_batch(self.data, self.labels, batch_size=batch_size)) :
loss = self.nn.train_on_batch(data[0], data[1],)
 losses.append(loss)
 valid = np.mean(np.abs(self.valid[1] - self.nn.predict(self.valid[0])))
 test_losses.append(np.mean(valid))
if batch_i%intervals == 0 or(batch_i == batch_size-1):
print(f"[Epoch {epoch}/{epochs}] [Batch {batch_i}] [MAE loss {loss}] [valid loss {valid}]")
 plt.plot(losses, label="training loss")
 plt.plot(test_losses, label="validation loss")
 plt.legend()
plt.show()
if epoch > 4:
stop =(test_losses[-1] > np.min(test_losses)+ 0.1)
 if stop:
print(f"stop train at [Epoch {epoch}/{epochs}]")
 break
def predict(self, test_data):
return self.nn.predict(test_data)
<compute_test_metric>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(LSTM(300, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
lstm = MyLSTM(X_train, y_train, X_test, y_test )<train_model>
model.fit(X_t, y, batch_size=32, epochs=2, validation_split=0.1);
lstm.train_on_batch(batch_size=50,epochs=50 )<load_from_csv>
y_test = model.predict([X_te], batch_size=1024, verbose=1 )
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 index = list(submission.index )<load_from_csv>
y_test[y_test >= 0.5] = 1
y_test[y_test < 0.5] = 0
y_test
test = []
for seg_id in index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 test.append(rolling_feature(seg['acoustic_data']))
test = np.array(test)
 test = scaler.transform(test)
<predict_on_test>
y_test = y_test.astype(int)
 y_test
y_pre2 = lstm.predict(test )<prepare_output>
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
y_pseudo_label = np.append(y, y_test, axis = 0)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_sentences_test = test["comment_text"].fillna("_na_" ).values
list_sentences_train_pseudo = np.append(list_sentences_train, list_sentences_test, axis = 0 )
y_pre = 0.6*y_pre1 + 0.4*y_pre<save_to_csv>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train_pseudo))
list_tokenized_train_pseudo = tokenizer.texts_to_sequences(list_sentences_train_pseudo)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t_pseudo = pad_sequences(list_tokenized_train_pseudo, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )
submission.time_to_failure = y_pre
submission.to_csv('submission.csv' )<import_modules>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split())for o in open(EMBEDDING_FILE))
import pandas as pd
import torch
import numpy as np
import lightgbm as lgb
import xgboost as xgb
from sklearn import preprocessing
from xgboost.sklearn import XGBClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn import svm
from sklearn.svm import SVC
from sklearn.ensemble import VotingClassifier
from sklearn.model_selection import cross_val_score
from sklearn.metrics import accuracy_score
from sklearn.neighbors import KNeighborsClassifier
from imblearn.over_sampling import SMOTE
from sklearn.ensemble import ExtraTreesClassifier
from xgboost import plot_importance
import matplotlib.pyplot as plt
import matplotlib.pylab as plt1<load_from_csv>
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
data = pd.read_csv('.. /input/train.csv',dtype={'duration': int})
 data_test = pd.read_csv('.. /input/test.csv',dtype={'duration': int})
 subs = pd.read_csv('.. /input/sample_submission.csv',dtype={'duration': int})
 data_csv1 = data 
<drop_column>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(LSTM(300, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
def comb(arr,data_csv):
test = []
for type in arr:
test.append(data_csv[type])
 data_csv.drop([type], axis=1,inplace=True)
 array2 = np.array(test)
 max_p_row= array2.argmax(axis=0)
 return max_p_row
norent=data_csv1[data_csv1['v2a1'].isnull() ]
print(norent.shape)
 own_house=norent[norent['tipovivi1']==1]['Id'].count()
installments=norent[norent['tipovivi2']==1]['Id'].count()
rented=norent[norent['tipovivi3']==1]['Id'].count()
Precarious=norent[norent['tipovivi4']==1]['Id'].count()
other= norent[norent['tipovivi5']==1]['Id'].count()
num_list=[own_house,installments,rented,Precarious,other]
name_list=['own_house','installments','rented','Precarious','other']
plt.bar(range(len(num_list)) ,num_list,tick_label=name_list)
 plt.show()
life_condition={'wall_material':['paredblolad','paredzocalo','paredpreb','pareddes','paredmad','paredzinc','paredfibras','paredother'],
'floor':['pisomoscer','pisocemento','pisoother','pisonatur','pisonotiene','pisomadera'],
'roof':['techozinc','techoentrepiso','techocane','techootro'],
'water':['abastaguadentro','abastaguafuera','abastaguan1o'],
'electricity':['public','planpri','noelec','coopele'],
'sanitario':['sanitario1','sanitario2','sanitario3','sanitario5','sanitario6'],
'energcocinar':['energcocinar1','energcocinar2','energcocinar3','energcocinar4'],
'etecho':['etecho2','etecho3'],
'eviv':['eviv1','eviv2','eviv3'],
'estadocivil':['estadocivil1','estadocivil2','estadocivil3','estadocivil4','estadocivil5','estadocivil6','estadocivil7'],
'parentesco':['parentesco1','parentesco2','parentesco3','parentesco4','parentesco5','parentesco6','parentesco7','parentesco8','parentesco9','parentesco10','parentesco11','parentesco12'],
'instlevel':['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9'],
'tipovivi':['tipovivi1','tipovivi2','tipovivi3','tipovivi4','tipovivi5'],
'area':['area1',"area2"],
}
for life in(life_condition):
_type = comb(life_condition[life], data_x)
 data_x.insert(2,life, _type)
 for life in(life_condition):
_type = comb(life_condition[life], test_data)
 test_data.insert(2,life, _type)
 kind = ['regular', 'borderline1', 'borderline2', 'svm']
sm = SMOTE(kind='svm')
 X_res, y_res = sm.fit_sample(data_x, data_y)
 columns = test_data.columns.values
X_res = pd.DataFrame(X_res,columns=columns)
 print(X_res)
 print(test_data)
<save_to_csv>
model.fit(X_t_pseudo, y_pseudo_label, batch_size=32, epochs=2, validation_split=0.1);
ans = model.predict(test_data)
 subs['Target'] = ans
print(subs)
 subs.to_csv('sample_submission.csv',index=False)
<import_modules>
y_test_final = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv')
 sample_submission[list_classes] = y_test_final
sample_submission.to_csv('submission.csv', index=False )