Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<import_modules>
%matplotlib inline
import os
import numpy as np
import pandas as pd
from sklearn import preprocessing
import xgboost as xgb<import_modules>
train1 = pd.read_csv(".. /input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv(".. /input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
print("XGBoost version:", xgb.__version__ )<define_variables>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']]
] )
print('
total_size = 0
start_path = '.. /input/jane-street-market-prediction'
for path, dirs, files in os.walk(start_path):
for f in files:
fp = os.path.join(path, f)
 total_size += os.path.getsize(fp)
 print("Directory size: " + str(round(total_size/ 1000000, 2)) + 'MB' )<load_from_csv>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1')
 ] ).sample(100000, random_state = 1)
 train = pd.concat([
train,
valid[["comment_text", "toxic"]]
] )
%%time
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 features = pd.read_csv('.. /input/jane-street-market-prediction/features.csv')
 example_test = pd.read_csv('.. /input/jane-street-market-prediction/example_test.csv')
 sample_prediction_df = pd.read_csv('.. /input/jane-street-market-prediction/example_sample_submission.csv')
 print("Data is loaded!" )<concatenate>
plt.figure(figsize =(12, 8))
msno.bar(train )
np.product(train.shape )<count_missing_values>
tokenizer = AutoTokenizer.from_pretrained("jplu/tf-xlm-roberta-large" )
missing_values_count = train.isnull().sum()
print(missing_values_count)
 total_cells = np.product(train.shape)
 total_missing = missing_values_count.sum()
print("% of missing data = ",(total_missing/total_cells)* 100 )<filter>
exp_text = "I am currently participating in Jigsaw competition"
tokenizer.tokenize(exp_text )
train[train['weight'] == 0]<prepare_x_and_y>
MAX_LEN = 192
train = train[train['weight'] != 0]
train['action'] =(( train['weight'].values * train['resp'].values)> 0 ).astype('int')
 X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = train.loc[:, 'action']<data_type_conversions>
def preprocess(data, max_seq_length = MAX_LEN, tokenizer = tokenizer):
ids = []
masks = []
segment = []
for i in tqdm(range(len(data))):
tokens = tokenizer.tokenize(data[i])
 if len(tokens)> max_seq_length - 2:
tokens = tokens[ : max_seq_length - 2]
input_ids = tokenizer.convert_tokens_to_ids(["[CLS]"] + tokens + ["[SEP]"])
 input_masks = [1] * len(input_ids)
 padding = max_seq_length - len(input_ids)
 input_ids.extend([0] * padding)
 input_masks.extend([0] * padding)
 segment_ids =[0]* max_seq_length
ids.append(input_ids)
 masks.append(input_masks)
 segment.append(segment_ids)
 return(np.array(ids), np.array(masks), np.array(segment))
X_train = X_train.fillna(0 )<drop_column>
train_ids, train_masks, train_segment = preprocess(train["comment_text"].values )
del train, x, y<train_model>
test_ids, test_masks, test_segment = preprocess(test["content"].values )
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=11,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.7,
missing=-999,
random_state=2020,
tree_method='gpu_hist'
)<train_model>
valid_ids, valid_masks, valid_segment = preprocess(valid["comment_text"].values )
%time clf.fit(X_train, y_train )<predict_on_test>
y_train = train["toxic"].values
y_valid = valid["toxic"].values
for(test_df, sample_prediction_df)in iter_test:
X_test = test_df.loc[:, test_df.columns.str.contains('feature')]
X_test.fillna(0)
 y_preds = clf.predict(X_test)
 sample_prediction_df.action = y_preds
env.predict(sample_prediction_df)
<set_options>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
warnings.filterwarnings('ignore')
 tf.random.set_seed(42)
<load_from_csv>
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv', nrows = 3)
 features = [c for c in train.columns if 'feature' in c] 
<choose_model_class>
def model(roberta_layer, max_len = MAX_LEN):
input_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 pooled_output, sequence_output = roberta_layer([input_word_ids, input_mask, segment_ids])
 x = pooled_output
x = tf.keras.layers.Flatten()(x)
 x = tf.keras.layers.Dense(128, activation = "relu" )(x)
 x = tf.keras.layers.Dense(1, activation = "sigmoid" )(x)
 model = tf.keras.Model(inputs = [input_word_ids, input_mask, segment_ids], outputs = x)
 return model
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model<prepare_x_and_y>
with strategy.scope() :
roberta_layer = TFAutoModel.from_pretrained("jplu/tf-xlm-roberta-large", trainable = True)
 model = model(roberta_layer)
 model.compile(tf.keras.optimizers.Adam(lr=1e-5), loss="binary_crossentropy", metrics=["accuracy"])
 model.summary()
batch_size = 4096
hidden_units = [384, 896, 896, 394]
dropout_rates = [0.10143786981358652, 0.19720339053599725, 0.2703017847244654, 0.23148340929571917, 0.2357768967777311]
label_smoothing = 1e-2
learning_rate = 1e-3 
<load_pretrained>
skf = StratifiedKFold(n_splits=5, shuffle = True)
 skf.get_n_splits(train_ids, y_train)
 skf
num_models = 2
models = []
for i in range(num_models):
clf = create_mlp(len(features), 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.load_weights(f'.. /input/js-nn-models/JSModel_{i}.hdf5')
 models.append(clf )<load_pretrained>
i = 1
preds = []
for train_index, test_index in skf.split(train_ids, y_train):
print("
")
 print("
print(f"FOLD No {i}")
 print("
tr_ids = train_ids[train_index]
tr_masks = train_masks[train_index]
tr_segment = train_segment[train_index]
vd_ids = train_ids[test_index]
vd_masks = train_masks[test_index]
vd_segment = train_segment[test_index]
y_tr = y_train[train_index]
y_vd = y_train[test_index]
history = model.fit(
(tr_ids, tr_masks, tr_segment), y_tr,
epochs=2,
batch_size=BATCH_SIZE,
validation_data =(( vd_ids, vd_masks, vd_segment), y_vd),
steps_per_epoch = len(tr_ids)//BATCH_SIZE)
 predictions = model.predict(( test_ids, test_masks, test_segment))
preds.append(predictions)
 i += 1
K.clear_session()
f_mean = np.load('.. /input/js-nn-models/f_mean.npy')
<split>
predictions =(preds[0] + preds[1] + preds[2] + preds[3] + preds[4])/5
<feature_engineering><EOS> 
sub["toxic"] = predictions
sub.set_index("id", inplace = True)
 sub.to_csv("submission.csv" )