Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<load_from_csv>
%matplotlib inline
%matplotlib inline
train_set = pd.read_csv('.. /input/train.csv')
 test_set = pd.read_csv('.. /input/test.csv')
 print(f'train set has {train_set.shape[0]} rows, and {train_set.shape[1]} features')
 print(f'test set has {test_set.shape[0]} rows, and {test_set.shape[1]} features' )<count_values>
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE=f'{path}glove6b50d/glove.6B.50d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv.zip'
TEST_DATA_FILE=f'{path}{comp}test.csv.zip'
TEST_LABELS_FILE = f'{path}{comp}test_labels.csv.zip'
target = train_set['Target']
target.value_counts(normalize=True )<feature_engineering>
embed_size = 50
max_features = 20000
maxlen = 200
test_set.loc[test_set['rez_esc'] == 99.0 , 'rez_esc'] = 5<create_dataframe>
train = pd.read_csv(TRAIN_DATA_FILE)
 test = pd.read_csv(TEST_DATA_FILE)
 test_l = pd.read_csv(TEST_LABELS_FILE )
data_na = train_set.isnull().sum().values / train_set.shape[0] *100
df_na = pd.DataFrame(data_na, index=train_set.columns, columns=['Count'])
 df_na = df_na.sort_values(by=['Count'], ascending=False)
 missing_value_count = df_na[df_na['Count']>0].shape[0]
print(f'We got {missing_value_count} rows which have missing value in train set ')
 df_na.head(6 )<create_dataframe>
train.isnull().any() ,test.isnull().any()
data_na = test_set.isnull().sum().values / test_set.shape[0] *100
df_na = pd.DataFrame(data_na, index=test_set.columns, columns=['Count'])
 df_na = df_na.sort_values(by=['Count'], ascending=False)
 missing_value_count = df_na[df_na['Count']>0].shape[0]
print(f'We got {missing_value_count} rows which have missing value in test set ')
 df_na.head(6 )<feature_engineering>
def clean_text(text):
text = text.lower()
text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r"'scuse", " excuse ", text)
 text = re.sub('\W', ' ', text)
 text = re.sub('\s+', ' ', text)
 text = re.sub(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ' _ip_ ', text)
 text = re.sub(r'(['"\.\(\)\!\?\-\\\/\,])', r' \1 ',text)
 text = re.sub(r'([\;\:\|•«
])', ' ', text)
 text = text.replace('&', ' and ')
 text = text.replace('@', ' at ')
 text = text.replace('0', ' zero ')
 text = text.replace('1', ' one ')
 text = text.replace('2', ' two ')
 text = text.replace('3', ' three ')
 text = text.replace('4', ' four ')
 text = text.replace('5', ' five ')
 text = text.replace('6', ' six ')
 text = text.replace('7', ' seven ')
 text = text.replace('8', ' eight ')
 text = text.replace('9', ' nine ')
 text = text.strip(' ')
 return text
def repalce_v18q1(x):
if x['v18q'] == 0:
return x['v18q']
else:
return x['v18q1']
train_set['v18q1'] = train_set.apply(lambda x : repalce_v18q1(x),axis=1)
 test_set['v18q1'] = test_set.apply(lambda x : repalce_v18q1(x),axis=1)
 train_set['v2a1'] = train_set['v2a1'].fillna(value=train_set['tipovivi3'])
 test_set['v2a1'] = test_set['v2a1'].fillna(value=test_set['tipovivi3'] )<data_type_conversions>
train['comment_text'] = train['comment_text'].map(lambda com : clean_text(com))
cols = ['edjefe', 'edjefa']
train_set[cols] = train_set[cols].replace({'no': 0, 'yes':1} ).astype(float)
 test_set[cols] = test_set[cols].replace({'no': 0, 'yes':1} ).astype(float )<categorify>
list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
list_sentences_test = test["comment_text"].fillna("_na_" ).values
train_set['roof_waste_material'] = np.nan
test_set['roof_waste_material'] = np.nan
train_set['electricity_other'] = np.nan
test_set['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
train_set['roof_waste_material'] = train_set.apply(lambda x : fill_roof_exception(x),axis=1)
 test_set['roof_waste_material'] = test_set.apply(lambda x : fill_roof_exception(x),axis=1)
 train_set['electricity_other'] = train_set.apply(lambda x : fill_no_electricity(x),axis=1)
 test_set['electricity_other'] = test_set.apply(lambda x : fill_no_electricity(x),axis=1 )<feature_engineering>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )
def owner_is_adult(x):
if x['age'] <= 19:
return 0
else:
return 1
train_set['head<18'] = train_set.apply(lambda x : owner_is_adult(x),axis=1)
 test_set['head<18'] = test_set.apply(lambda x : owner_is_adult(x),axis=1 )<feature_engineering>
train_set['adult'] = train_set['hogar_adul'] - train_set['hogar_mayor']
train_set['dependency_count'] = train_set['hogar_nin'] + train_set['hogar_mayor']
train_set['dependency'] = train_set['dependency_count'] / train_set['adult']
train_set['child_percent'] = train_set['hogar_nin']/train_set['hogar_total']
train_set['elder_percent'] = train_set['hogar_mayor']/train_set['hogar_total']
train_set['adult_percent'] = train_set['hogar_adul']/train_set['hogar_total']
test_set['adult'] = test_set['hogar_adul'] - test_set['hogar_mayor']
test_set['dependency_count'] = test_set['hogar_nin'] + test_set['hogar_mayor']
test_set['dependency'] = test_set['dependency_count'] / test_set['adult']
test_set['child_percent'] = test_set['hogar_nin']/test_set['hogar_total']
test_set['elder_percent'] = test_set['hogar_mayor']/test_set['hogar_total']
test_set['adult_percent'] = test_set['hogar_adul']/test_set['hogar_total']
train_set['rent_per_adult'] = train_set['v2a1']/train_set['hogar_adul']
train_set['rent_per_person'] = train_set['v2a1']/train_set['hhsize']
test_set['rent_per_adult'] = test_set['v2a1']/test_set['hogar_adul']
test_set['rent_per_person'] = test_set['v2a1']/test_set['hhsize']
train_set['overcrowding_room_and_bedroom'] =(train_set['hacdor'] + train_set['hacapo'])/2
test_set['overcrowding_room_and_bedroom'] =(test_set['hacdor'] + test_set['hacapo'])/2
train_set['no_appliances'] = train_set['refrig'] + train_set['computer'] + train_set['television']
test_set['no_appliances'] = test_set['refrig'] + test_set['computer'] + test_set['television']
train_set['r4h1_percent_in_male'] = train_set['r4h1'] / train_set['r4h3']
train_set['r4m1_percent_in_female'] = train_set['r4m1'] / train_set['r4m3']
train_set['r4h1_percent_in_total'] = train_set['r4h1'] / train_set['hhsize']
train_set['r4m1_percent_in_total'] = train_set['r4m1'] / train_set['hhsize']
train_set['r4t1_percent_in_total'] = train_set['r4t1'] / train_set['hhsize']
test_set['r4h1_percent_in_male'] = test_set['r4h1'] / test_set['r4h3']
test_set['r4m1_percent_in_female'] = test_set['r4m1'] / test_set['r4m3']
test_set['r4h1_percent_in_total'] = test_set['r4h1'] / test_set['hhsize']
test_set['r4m1_percent_in_total'] = test_set['r4m1'] / test_set['hhsize']
test_set['r4t1_percent_in_total'] = test_set['r4t1'] / test_set['hhsize']
train_set['rent_per_room'] = train_set['v2a1']/train_set['rooms']
train_set['bedroom_per_room'] = train_set['bedrooms']/train_set['rooms']
train_set['elder_per_room'] = train_set['hogar_mayor']/train_set['rooms']
train_set['adults_per_room'] = train_set['adult']/train_set['rooms']
train_set['child_per_room'] = train_set['hogar_nin']/train_set['rooms']
train_set['male_per_room'] = train_set['r4h3']/train_set['rooms']
train_set['female_per_room'] = train_set['r4m3']/train_set['rooms']
train_set['room_per_person_household'] = train_set['hhsize']/train_set['rooms']
test_set['rent_per_room'] = test_set['v2a1']/test_set['rooms']
test_set['bedroom_per_room'] = test_set['bedrooms']/test_set['rooms']
test_set['elder_per_room'] = test_set['hogar_mayor']/test_set['rooms']
test_set['adults_per_room'] = test_set['adult']/test_set['rooms']
test_set['child_per_room'] = test_set['hogar_nin']/test_set['rooms']
test_set['male_per_room'] = test_set['r4h3']/test_set['rooms']
test_set['female_per_room'] = test_set['r4m3']/test_set['rooms']
test_set['room_per_person_household'] = test_set['hhsize']/test_set['rooms']
train_set['rent_per_bedroom'] = train_set['v2a1']/train_set['bedrooms']
train_set['edler_per_bedroom'] = train_set['hogar_mayor']/train_set['bedrooms']
train_set['adults_per_bedroom'] = train_set['adult']/train_set['bedrooms']
train_set['child_per_bedroom'] = train_set['hogar_nin']/train_set['bedrooms']
train_set['male_per_bedroom'] = train_set['r4h3']/train_set['bedrooms']
train_set['female_per_bedroom'] = train_set['r4m3']/train_set['bedrooms']
train_set['bedrooms_per_person_household'] = train_set['hhsize']/train_set['bedrooms']
test_set['rent_per_bedroom'] = test_set['v2a1']/test_set['bedrooms']
test_set['edler_per_bedroom'] = test_set['hogar_mayor']/test_set['bedrooms']
test_set['adults_per_bedroom'] = test_set['adult']/test_set['bedrooms']
test_set['child_per_bedroom'] = test_set['hogar_nin']/test_set['bedrooms']
test_set['male_per_bedroom'] = test_set['r4h3']/test_set['bedrooms']
test_set['female_per_bedroom'] = test_set['r4m3']/test_set['bedrooms']
test_set['bedrooms_per_person_household'] = test_set['hhsize']/test_set['bedrooms']
train_set['tablet_per_person_household'] = train_set['v18q1']/train_set['hhsize']
train_set['phone_per_person_household'] = train_set['qmobilephone']/train_set['hhsize']
test_set['tablet_per_person_household'] = test_set['v18q1']/test_set['hhsize']
test_set['phone_per_person_household'] = test_set['qmobilephone']/test_set['hhsize']
train_set['age_12_19'] = train_set['hogar_nin'] - train_set['r4t1']
test_set['age_12_19'] = test_set['hogar_nin'] - test_set['r4t1']
train_set['escolari_age'] = train_set['escolari']/train_set['age']
test_set['escolari_age'] = test_set['escolari']/test_set['age']
train_set['rez_esc_escolari'] = train_set['rez_esc']/train_set['escolari']
train_set['rez_esc_r4t1'] = train_set['rez_esc']/train_set['r4t1']
train_set['rez_esc_r4t2'] = train_set['rez_esc']/train_set['r4t2']
train_set['rez_esc_r4t3'] = train_set['rez_esc']/train_set['r4t3']
train_set['rez_esc_age'] = train_set['rez_esc']/train_set['age']
test_set['rez_esc_escolari'] = test_set['rez_esc']/test_set['escolari']
test_set['rez_esc_r4t1'] = test_set['rez_esc']/test_set['r4t1']
test_set['rez_esc_r4t2'] = test_set['rez_esc']/test_set['r4t2']
test_set['rez_esc_r4t3'] = test_set['rez_esc']/test_set['r4t3']
test_set['rez_esc_age'] = test_set['rez_esc']/test_set['age']<feature_engineering>
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
train_set['dependency'] = train_set['dependency'].replace({np.inf: 0})
 test_set['dependency'] = test_set['dependency'].replace({np.inf: 0})
 print(f'train set has {train_set.shape[0]} rows, and {train_set.shape[1]} features')
 print(f'test set has {test_set.shape[0]} rows, and {test_set.shape[1]} features' )<create_dataframe>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(50, return_sequences=True))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAvgPool1D()(x)
 max_pool = GlobalMaxPool1D()(x)
 conc = concatenate([avg_pool,max_pool])
 x = GlobalMaxPool1D()(x)
 x = Dense(50, activation='elu' )(x)
 x = Dropout(0.25 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
df_train = pd.DataFrame()
df_test = pd.DataFrame()
aggr_mean_list = ['rez_esc', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7', 'parentesco2',
'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',]
other_list = ['escolari', 'age', 'escolari_age']
for item in aggr_mean_list:
group_train_mean = train_set[item].groupby(train_set['idhogar'] ).mean()
group_test_mean = test_set[item].groupby(test_set['idhogar'] ).mean()
new_col = item + '_aggr_mean'
df_train[new_col] = group_train_mean
df_test[new_col] = group_test_mean
for item in other_list:
for function in ['mean','std','min','max','sum']:
group_train = train_set[item].groupby(train_set['idhogar'] ).agg(function)
 group_test = test_set[item].groupby(test_set['idhogar'] ).agg(function)
 new_col = item + '_' + function
df_train[new_col] = group_train
df_test[new_col] = group_test
print(f'new aggregate train set has {df_train.shape[0]} rows, and {df_train.shape[1]} features')
 print(f'new aggregate test set has {df_test.shape[0]} rows, and {df_test.shape[1]} features' )<merge>
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Flatten, Dense, Dropout, Activation, LSTM, Bidirectional, GlobalMaxPool1D, Embedding,AveragePooling1D
from keras import Input, Model
from keras import layers, initializers, regularizers, constraints, optimizers
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
df_test = df_test.reset_index()
df_train = df_train.reset_index()
train_agg = pd.merge(train_set, df_train, on='idhogar')
 test = pd.merge(test_set, df_test, on='idhogar')
 train_agg.fillna(value=0, inplace=True)
 test.fillna(value=0, inplace=True)
 print(f'new train set has {train_agg.shape[0]} rows, and {train_agg.shape[1]} features')
 print(f'new test set has {test.shape[0]} rows, and {test.shape[1]} features' )<filter>
file_path = "bi_lstm.best.hdf5"
checkpoint = ModelCheckpoint(file_path, monitor='val_loss',verbose=1, save_best_only=True, mode = 'min')
 early = EarlyStopping(monitor="val_loss", mode="min",patience=1 )
train = train_agg.query('parentesco1==1' )<sort_values>
batch_size = 32
epochs = 10
callbacks_list = [checkpoint,early]
history = model.fit(X_t,y, batch_size=batch_size, epochs=epochs, validation_split=0.1, callbacks=callbacks_list )
submission = test[['Id']]
train.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 test.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 correlation = train.corr()
correlation = correlation['Target'].sort_values(ascending=False)
 print(f'The most 20 positive feature:
{correlation.head(20)}')
 print('*'*50)
 print(f'The most 20 negative feature:
{correlation.tail(20)}' )<prepare_x_and_y>
history.history
y = train['Target']
train.drop(columns=['Target'], inplace=True)
 clf = lgb.LGBMClassifier(max_depth=9, learning_rate=0.01, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=2500, class_weight='balanced',
colsample_bytree = 0.93, min_child_samples = 20, num_leaves = 21, subsample = 0.96 )<train_model>
y_test = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv.zip')
 sample_submission[list_classes] = y_test
sample_submission.to_csv('submission.csv', index=False )
<save_to_csv><EOS> 
print("end" )