Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<data_type_conversions>
MAX_LEN = 192
DROPOUT = 0.5000001
BATCH_SIZE = 16
TOTAL_STEPS_STAGE1 = 2000
VALIDATE_EVERY_STAGE1 = 200
TOTAL_STEPS_STAGE2 = 200
VALIDATE_EVERY_STAGE2 = 10
LR_TRANSFORMER = 5e-6
LR_HEAD = 1e-3
PRETRAINED_TOKENIZER= 'jplu/tf-xlm-roberta-large'
PRETRAINED_MODEL = '/kaggle/input/jigsaw-mlm-finetuned-xlm-r-large'
D = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'
D_TRANS = '/kaggle/input/jigsaw-train-multilingual-coments-google-api/'
print(tf.__version__)
 logging.getLogger().setLevel(logging.NOTSET)
 AUTO = tf.data.experimental.AUTOTUNE
train[features[1:]] = train[features[1:]].fillna(f_mean)
 train['action'] =(train['resp'] > 0 ).astype('int')
 np.save('f_mean.npy', f_mean )<choose_model_class>
def connect_to_TPU() :

 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
global_batch_size = BATCH_SIZE * strategy.num_replicas_in_sync
return tpu, strategy, global_batch_size
tpu, strategy, global_batch_size = connect_to_TPU()
print("REPLICAS: ", strategy.num_replicas_in_sync )
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model<prepare_x_and_y>
def load_jigsaw_trans(langs=['tr','it','es','ru','fr','pt'],
columns=['comment_text', 'toxic']):
train_6langs=[]
for i in range(len(langs)) :
fn = D_TRANS+'jigsaw-toxic-comment-train-google-%s-cleaned.csv'%langs[i]
train_6langs.append(downsample(pd.read_csv(fn)[columns]))
return train_6langs
def downsample(df):

 ds_df= pd.concat([
df.query('toxic==1'),
df.query('toxic==0' ).sample(sum(df.toxic))
])
 return ds_df
train_df = pd.concat(load_jigsaw_trans())
 val_df = pd.read_csv(D+'validation.csv')
 test_df = pd.read_csv(D+'test.csv')
 sub_df = pd.read_csv(D+'sample_submission.csv' )
batch_size = 4096
hidden_units = [384, 896, 896, 394]
dropout_rates = [
0.10143786981358652,
0.19720339053599725,
0.2703017847244654,
0.23148340929571917,
0.2357768967777311,
]
label_smoothing = 1e-2
learning_rate = 1e-7
oof = np.zeros(len(train['action']))
gkf = GroupKFold(n_splits = 5)
 for fold,(tr, te)in enumerate(gkf.split(train['action'].values, train['action'].values, train['date'].values)) :
X_tr, X_val = train.loc[tr, features].values, train.loc[te, features].values
y_tr, y_val = train.loc[tr, 'action'].values, train.loc[te, 'action'].values
ckp_path = f'JSModel_{fold}.hdf5'
model = create_mlp(X_tr.shape[1], 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 rlr = ReduceLROnPlateau(monitor = 'val_AUC', factor = 0.1, patience = 3, verbose = 0,
min_delta = 1e-4, mode = 'max')
 ckp = ModelCheckpoint(ckp_path, monitor = 'val_AUC', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'max')
 es = EarlyStopping(monitor = 'val_AUC', min_delta = 1e-4, patience = 7, mode = 'max',
baseline = None, restore_best_weights = True, verbose = 0)
 model.fit(X_tr, y_tr, validation_data =(X_val, y_val), epochs = 1000,
batch_size = batch_size, callbacks = [rlr, ckp, es], verbose = 0)
 oof[te] += model.predict(X_val, batch_size = batch_size * 4 ).ravel()
score = roc_auc_score(y_val, oof[te])
 print(f'Fold {fold} ROC AUC:\t', score)
 model = create_mlp(X_tr.shape[1], 1, hidden_units, dropout_rates, label_smoothing, learning_rate / 100)
 model.load_weights(ckp_path)
 model.fit(X_val, y_val, epochs = 4, batch_size = batch_size, verbose = 0)
 model.save_weights(ckp_path )<compute_test_metric>
%%time
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
 tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_TOKENIZER)
 X_train = regular_encode(train_df.comment_text.values, tokenizer, maxlen=MAX_LEN)
 X_val = regular_encode(val_df.comment_text.values, tokenizer, maxlen=MAX_LEN)
 X_test = regular_encode(test_df.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train_df.toxic.values.reshape(-1,1)
 y_val = val_df.toxic.values.reshape(-1,1 )
score_oof = roc_auc_score(train['action'].values, oof)
 print(score_oof )<load_pretrained>
def create_dist_dataset(X, y=None, training=False):
dataset = tf.data.Dataset.from_tensor_slices(X)
 if y is not None:
dataset_y = tf.data.Dataset.from_tensor_slices(y)
 dataset = tf.data.Dataset.zip(( dataset, dataset_y))
if training:
dataset = dataset.shuffle(len(X)).repeat()
dataset = dataset.batch(global_batch_size ).prefetch(AUTO)
 dist_dataset = strategy.experimental_distribute_dataset(dataset)
 return dist_dataset
train_dist_dataset = create_dist_dataset(X_train, y_train, True)
 val_dist_dataset = create_dist_dataset(X_val)
 test_dist_dataset = create_dist_dataset(X_test )
num_models = 2
models = []
for i in range(num_models):
clf = create_mlp(len(features), 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.load_weights(f'./JSModel_{i}.hdf5')
 models.append(clf)
<load_pretrained>
%%time
def create_model_and_optimizer() :
with strategy.scope() :
transformer_layer = TFRobertaModel.from_pretrained(PRETRAINED_MODEL)
 model = build_model(transformer_layer)
 optimizer_transformer = Adam(learning_rate=LR_TRANSFORMER)
 optimizer_head = Adam(learning_rate=LR_HEAD)
 return model, optimizer_transformer, optimizer_head
def build_model(transformer):
inp = Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_word_ids")
 x = transformer(inp)[0][:, 0, :]
x = Dropout(DROPOUT )(x)
 out = Dense(1, activation='sigmoid', name='custom_head' )(x)
 model = Model(inputs=[inp], outputs=[out])
 return model
model, optimizer_transformer, optimizer_head = create_model_and_optimizer()
model.summary()
f_mean = np.load('./f_mean.npy')
<split>
def define_losses_and_metrics() :
with strategy.scope() :
loss_object = tf.keras.losses.BinaryCrossentropy(
 reduction=tf.keras.losses.Reduction.NONE, from_logits=False)
 def compute_loss(labels, predictions):
per_example_loss = loss_object(labels, predictions)
 loss = tf.nn.compute_average_loss(
 per_example_loss, global_batch_size = global_batch_size)
 return loss
train_accuracy_metric = tf.keras.metrics.AUC(name='training_AUC')
 return compute_loss, train_accuracy_metric
def train(train_dist_dataset, val_dist_dataset=None, y_val=None,
total_steps=2000, validate_every=200):
best_weights, history = None, []
step = 0
for tensor in train_dist_dataset:
distributed_train_step(tensor)
 step+=1
if(step % validate_every == 0):
train_metric = train_accuracy_metric.result().numpy()
print("Step %d, train AUC: %.5f" %(step, train_metric))
if val_dist_dataset:
val_metric = roc_auc_score(y_val, predict(val_dist_dataset))
print("Step %d, val AUC: %.5f" %(step,val_metric))
history.append(val_metric)
 if history[-1] == max(history):
best_weights = model.get_weights()
train_accuracy_metric.reset_states()
if step == total_steps:
break
model.set_weights(best_weights)
 @tf.function
def distributed_train_step(data):
strategy.experimental_run_v2(train_step, args=(data,))
def train_step(inputs):
features, labels = inputs
transformer_trainable_variables = [ v for v in model.trainable_variables
if(( 'pooler' not in v.name)and 
('custom' not in v.name)) ]
head_trainable_variables = [ v for v in model.trainable_variables
if 'custom' in v.name]
with tf.GradientTape(persistent=True)as tape:
predictions = model(features, training=True)
 loss = compute_loss(labels, predictions)
 gradients_transformer = tape.gradient(loss, transformer_trainable_variables)
 gradients_head = tape.gradient(loss, head_trainable_variables)
 del tape
optimizer_transformer.apply_gradients(zip(gradients_transformer,
transformer_trainable_variables))
optimizer_head.apply_gradients(zip(gradients_head,
head_trainable_variables))
train_accuracy_metric.update_state(labels, predictions)
 def predict(dataset):
predictions = []
for tensor in dataset:
predictions.append(distributed_prediction_step(tensor))
predictions = np.vstack(list(map(np.vstack,predictions)))
 return predictions
@tf.function
def distributed_prediction_step(data):
predictions = strategy.experimental_run_v2(prediction_step, args=(data,))
return strategy.experimental_local_results(predictions)
 def prediction_step(inputs):
features = inputs
predictions = model(features, training=False)
 return predictions
compute_loss, train_accuracy_metric = define_losses_and_metrics()
env = janestreet.make_env()
env_iter = env.iter_test()<categorify>
%%time
train(train_dist_dataset, val_dist_dataset, y_val,
TOTAL_STEPS_STAGE1, VALIDATE_EVERY_STAGE1 )
opt_th = 0.502
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = 0.
 for clf in models:
pred += clf(x_tt, training = False ).numpy().item() / num_models
pred_df.action = np.where(pred >= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<set_options>
%%time
optimizer_head.learning_rate.assign(1e-4)
 X_train, X_val, y_train, y_val = train_test_split(X_val, y_val, test_size = 0.1)
 train_dist_dataset = create_dist_dataset(X_train, y_train, training=True)
 val_dist_dataset = create_dist_dataset(X_val, y_val)
 train(train_dist_dataset, val_dist_dataset, y_val,
total_steps = TOTAL_STEPS_STAGE2,
validate_every = VALIDATE_EVERY_STAGE2 )
<define_variables><EOS> 
%%time
sub_df['toxic'] = predict(test_dist_dataset)[:,0]
sub_df.to_csv('submission.csv', index=False )