Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<drop_column>
!pip install -q nltk
!pip install -q WordCloud
!pip install -q plotly
!pip install -q transformers
!pip install -q twython
df_train.drop(cols_with_only_one_value, axis=1, inplace=True)
 df_test.drop(cols_with_only_one_value, axis=1, inplace=True )<sort_values>
nltk.download('stopwords')
 nltk.download('punkt')
 nltk.download('vader_lexicon')
cols_train = np.array(sorted([col for col in df_train.columns if col != 'Target']))
cols_test = np.array(sorted(df_test.columns))<create_dataframe>
dir_path = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'
train_data_1 = pd.read_csv(dir_path + "/jigsaw-toxic-comment-train.csv")
 train_data_2 = pd.read_csv(dir_path + "/jigsaw-unintended-bias-train.csv")
 validation_data = pd.read_csv(dir_path + "/validation.csv")
 test_data = pd.read_csv(dir_path + "/test.csv" )
agg_train = pd.DataFrame()
agg_test = pd.DataFrame()
for item in tqdm(family_size_features):
for i, function in enumerate(['mean','std','min','max','sum', 'count', max_min]):
group_train = df_train[item].groupby(df_train['idhogar'] ).agg(function)
 group_test = df_test[item].groupby(df_test['idhogar'] ).agg(function)
 if i == 6:
new_col = item + '_new_' + 'max_min'
else:
new_col = item + '_new_' + function
agg_train[new_col] = group_train
agg_test[new_col] = group_test
print('new aggregate train set has {} rows, and {} features'.format(agg_train.shape[0], agg_train.shape[1]))
print('new aggregate test set has {} rows, and {} features'.format(agg_test.shape[0], agg_test.shape[1]))<define_variables>
train_data_2.toxic = train_data_2.toxic.round().astype(int)
 train_data = pd.concat([
train_data_1[['comment_text','toxic']]
, train_data_2[['comment_text','toxic']].query('toxic == 1')
 , train_data_2[['comment_text', 'toxic']].query('toxic == 0' ).sample(n = 200000, random_state = 1993)
 ])
aggr_list = ['rez_esc', 'dis', 'male', 'female',
'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',
'epared1', 'epared2', 'epared3', 'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3', 'refrig', 'television', 'mobilephone',
'area1', 'area2', 'v18q', 'edjef']
for item in tqdm(aggr_list):
for function in ['count', 'sum']:
group_train = df_train[item].groupby(df_train['idhogar'] ).agg(function)
 group_test = df_test[item].groupby(df_test['idhogar'] ).agg(function)
 new_col = item + '_new1_' + function
agg_train[new_col] = group_train
agg_test[new_col] = group_test
print('new aggregate train set has {} rows, and {} features'.format(agg_train.shape[0], agg_train.shape[1]))
print('new aggregate test set has {} rows, and {} features'.format(agg_test.shape[0], agg_test.shape[1]))<groupby>
word_tokenize(train_data.comment_text[1])
aggr_list = ['escolari', 'age', 'escolari_age', 'dependency', 'bedrooms', 'overcrowding', 'rooms', 'qmobilephone', 'v18q1']
for item in tqdm(aggr_list):
for function in ['mean','std','min','max','sum', 'count', max_min]:
group_train = df_train[item].groupby(df_train['idhogar'] ).agg(function)
 group_test = df_test[item].groupby(df_test['idhogar'] ).agg(function)
 if i == 6:
new_col = item + '_new2_' + 'max_min'
else:
new_col = item + '_new2_' + function
agg_train[new_col] = group_train
agg_test[new_col] = group_test
print('new aggregate train set has {} rows, and {} features'.format(agg_train.shape[0], agg_train.shape[1]))
print('new aggregate test set has {} rows, and {} features'.format(agg_test.shape[0], agg_test.shape[1]))<merge>
text_words = word_tokenize(train_data.comment_text[1])
 text_words = [word.lower() for word in text_words]
agg_test = agg_test.reset_index()
agg_train = agg_train.reset_index()
train_agg = pd.merge(df_train, agg_train, on='idhogar')
 test = pd.merge(df_test, agg_test, on='idhogar')
 train_agg.fillna(value=0, inplace=True)
 test.fillna(value=0, inplace=True)
 print('train shape:', train_agg.shape, 'test shape:', test.shape )<define_variables>
modified_stopwords = stopwords.words('english')
 modified_stopwords.remove('not')
 text_words = [word for word in text_words if not word in modified_stopwords and word.isalpha() ]
len(text_words )
aggr_list = ['rez_esc', 'dis', 'male', 'female',
'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',
'epared1', 'epared2', 'epared3', 'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3', 'refrig', 'television', 'mobilephone',
'area1', 'area2', 'v18q', 'edjef']
for lugar in ['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']:
group_train = df_train[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).sum().reset_index()
group_train.columns = [lugar, 'idhogar'] + ['new3_{}_idhogar_{}'.format(lugar, col)for col in group_train][2:]
group_test = df_test[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).sum().reset_index()
group_test.columns = [lugar, 'idhogar'] + ['new3_{}_idhogar_{}'.format(lugar, col)for col in group_test][2:]
train_agg = pd.merge(train_agg, group_train, on=[lugar, 'idhogar'])
 test = pd.merge(test, group_test, on=[lugar, 'idhogar'])
 print('train shape:', train_agg.shape, 'test shape:', test.shape )<define_variables>
def nan_filter(x):
if type(x)== str:
return(x.replace("
", "")).lower()
else:
return ""
nontoxic_text = ' '.join([nan_filter(comment)for comment in train_sample.query('toxic==0')['comment_text']])
 toxic_text = ' '.join([nan_filter(comment)for comment in train_sample.query('toxic == 1')['comment_text']] )
aggr_list = ['rez_esc', 'dis', 'male', 'female',
'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',
'epared1', 'epared2', 'epared3', 'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3', 'refrig', 'television', 'mobilephone',
'area1', 'area2', 'v18q', 'edjef']
for lugar in ['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']:
group_train = df_train[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).sum().reset_index()
group_train.columns = [lugar, 'idhogar'] + ['new4_{}_idhogar_{}'.format(lugar, col)for col in group_train][2:]
group_test = df_test[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).sum().reset_index()
group_test.columns = [lugar, 'idhogar'] + ['new4_{}_idhogar_{}'.format(lugar, col)for col in group_test][2:]
train_agg = pd.merge(train_agg, group_train, on=[lugar, 'idhogar'])
 test = pd.merge(test, group_test, on=[lugar, 'idhogar'])
 print('train shape:', train_agg.shape, 'test shape:', test.shape )<groupby>
def text_len(x):
if type(x)is str:
return len(x.split())
 else:
return 0
train_sample['comment_size'] = train_sample.comment_text.apply(text_len)
 toxic_text_lengths = train_sample.query('toxic == 1 and comment_size < 200')['comment_size'].sample(frac = 1, random_state = 1993)
 nontoxic_text_lengths = train_sample.query('toxic == 0 and comment_size < 200')['comment_size'].sample(frac = 1, random_state = 1993)
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
for function in tqdm(['mean','std','min','max','sum', 'count', max_min]):
for lugar in ['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']:
group_train = df_train[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).agg(function ).reset_index()
group_train.columns = [lugar, 'idhogar'] + ['new5_{}_idhogar_{}_{}'.format(lugar, col, function)for col in group_train][2:]
group_test = df_test[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).agg(function ).reset_index()
group_test.columns = [lugar, 'idhogar'] + ['new5_{}_idhogar_{}_{}'.format(lugar, col, function)for col in group_test][2:]
train_agg = pd.merge(train_agg, group_train, on=[lugar, 'idhogar'])
 test = pd.merge(test, group_test, on=[lugar, 'idhogar'])
 print('train shape:', train_agg.shape, 'test shape:', test.shape )<filter>
def sentiment(x):
if type(x)is str:
return SIA.polarity_scores(x)
 else:
return 1000
SIA = SentimentIntensityAnalyzer()
train_sample['polarity'] = train_sample.comment_text.apply(sentiment)
train = train_agg.query('parentesco1==1' )<categorify>
train_sample['negativity'] = train_sample.polarity.apply(lambda x: x['neg'])
 train_sample['positivity'] = train_sample.polarity.apply(lambda x: x['pos'] )
train['dependency'].replace(np.inf, 0, inplace=True)
 test['dependency'].replace(np.inf, 0, inplace=True )<sort_values>
roberta_string = 'jplu/tf-xlm-roberta-large'
submission = test[['Id']]
train.drop(columns=['idhogar','Id', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 test.drop(columns=['idhogar','Id', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 correlation = train.corr()
correlation = correlation['Target'].sort_values(ascending=False )<count_values>
tokenizer = AutoTokenizer.from_pretrained(roberta_string )
binary_cat_features = [col for col in train.columns if train[col].value_counts().shape[0] == 2]
object_features = ['edjefe', 'edjefa']
categorical_feats = binary_cat_features + object_features<compute_test_metric>
sample_encoded = tokenizer.batch_encode_plus(train_sample.comment_text.values[0:2]
, return_attention_masks=False
, return_token_type_ids=False
, pad_to_max_length=True
, max_length = 512)
 sample_encoded
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True )<prepare_x_and_y>
def encode(text, max_len = 512):
encoded_dict = tokenizer.batch_encode_plus(text
, return_attention_masks=False
, return_token_type_ids=False
, pad_to_max_length=True
, max_length = max_len)
 return np.array(encoded_dict['input_ids'])
 MAX_LEN = 192
y = train['Target']
train.drop(columns=['Target'], inplace=True )<categorify>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
def print_execution_time(start):
end = time.time()
hours, rem = divmod(end-start, 3600)
 minutes, seconds = divmod(rem, 60)
 print('*'*20, "Execution ended in {:0>2}h {:0>2}m {:05.2f}s".format(int(hours),int(minutes),seconds), '*'*20 )<choose_model_class>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
def extract_good_features_using_shap_LGB(params, SEED):
clf = lgb.LGBMClassifier(objective='multiclass',
random_state=1989,
max_depth=params['max_depth'],
learning_rate=params['learning_rate'],
silent=True,
metric='multi_logloss',
n_jobs=-1, n_estimators=10000,
class_weight='balanced',
colsample_bytree = params['colsample_bytree'],
min_split_gain= params['min_split_gain'],
bagging_freq = params['bagging_freq'],
min_child_weight=params['min_child_weight'],
num_leaves = params['num_leaves'],
subsample = params['subsample'],
reg_alpha= params['reg_alpha'],
reg_lambda= params['reg_lambda'],
num_class=len(np.unique(y)) ,
bagging_seed=SEED,
seed=SEED, 
)
 kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 feat_importance_df = pd.DataFrame()
for i,(train_index, test_index)in enumerate(kf.split(train, y)) :
print('='*30, '{} of {} folds'.format(i+1, kfold), '='*30)
 start = time.time()
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric=evaluate_macroF1_lgb, categorical_feature=categorical_feats,
early_stopping_rounds=500, verbose=500)
 shap_values = shap.TreeExplainer(clf.booster_ ).shap_values(X_train)
 fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = X_train.columns
fold_importance_df['shap_values'] = abs(np.array(shap_values)[:, :].mean(1 ).mean(0))
fold_importance_df['feat_imp'] = clf.feature_importances_
feat_importance_df = pd.concat([feat_importance_df, fold_importance_df])
 print_execution_time(start)
 feat_importance_df_shap = feat_importance_df.groupby('feature' ).mean().sort_values('shap_values', ascending=False ).reset_index()
return feat_importance_df_shap<define_variables>
X_train = encode(train_data.comment_text.values, MAX_LEN)
 X_valid = encode(validation_data.comment_text.values, MAX_LEN)
 X_test = encode(test_data.content.values, MAX_LEN)
 y_train = train_data.toxic.values
y_valid = validation_data.toxic.values
total_shap_df = pd.DataFrame()
NUM_ITERATIONS = 50
for SEED in range(NUM_ITERATIONS):
print('
params = {'max_depth': np.random.choice([5, 6, 7, 8, 10, 12, -1]),
'learning_rate': np.random.rand() * 0.02,
'colsample_bytree': np.random.rand() *(1 - 0.5)+ 0.5,
'subsample': np.random.rand() *(1 - 0.5)+ 0.5,
'min_split_gain': np.random.rand() * 0.2,
'num_leaves': np.random.choice([32, 48, 64]),
'reg_alpha': np.random.rand() * 2,
'reg_lambda': np.random.rand() *2,
'bagging_freq': np.random.randint(4)+1,
'min_child_weight': np.random.randint(100)+ 20
}
temp_shap_df = extract_good_features_using_shap_LGB(params, SEED)
 total_shap_df = pd.concat([total_shap_df, temp_shap_df] )<create_dataframe>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( X_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( X_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(X_test)
.batch(BATCH_SIZE)
 )
new_train = train[top_features].copy()
new_test = test[top_features].copy()<define_variables>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(roberta_string)
 input_word_ids = Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer_layer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 model.summary()
new_categorical_feats = [col for col in top_features if col in categorical_feats]<choose_model_class>
n_steps = X_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
def LGB_OOF(params, categorical_feats, N_FOLDs, SEED=1989):
clf = lgb.LGBMClassifier(objective='multiclass',
random_state=1989,
max_depth=params['max_depth'],
learning_rate=params['learning_rate'],
silent=True,
metric='multi_logloss',
n_jobs=-1, n_estimators=10000,
class_weight='balanced',
colsample_bytree = params['colsample_bytree'],
min_split_gain= params['min_split_gain'],
bagging_freq = params['bagging_freq'],
min_child_weight=params['min_child_weight'],
num_leaves = params['num_leaves'],
subsample = params['subsample'],
reg_alpha= params['reg_alpha'],
reg_lambda= params['reg_lambda'],
num_class=len(np.unique(y)) ,
bagging_seed=SEED,
seed=SEED, 
)
 kfold = 10
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 feat_importance_df = pd.DataFrame()
predicts_result = []
for i,(train_index, test_index)in enumerate(kf.split(new_train, y)) :
print('='*30, '{} of {} folds'.format(i+1, kfold), '='*30)
 start = time.time()
X_train, X_val = new_train.iloc[train_index], new_train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric=evaluate_macroF1_lgb,categorical_feature=new_categorical_feats,
early_stopping_rounds=500, verbose=500)
 shap_values = shap.TreeExplainer(clf.booster_ ).shap_values(X_train)
 fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = X_train.columns
fold_importance_df['shap_values'] = abs(np.array(shap_values)[:, :].mean(1 ).mean(0))
fold_importance_df['feat_imp'] = clf.feature_importances_
feat_importance_df = pd.concat([feat_importance_df, fold_importance_df])
 predicts_result.append(clf.predict(new_test))
print_execution_time(start)
 return predicts_result, feat_importance_df<choose_model_class>
n_steps = X_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
<save_to_csv><EOS> 
submission = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 submission['toxic'] = model.predict(test_dataset, verbose = 1)
 submission.to_csv('/kaggle/working/submission.csv', index = False )