Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<prepare_x_and_y>
MAX_LEN = 192
DROPOUT = 0.5
BATCH_SIZE = 16
TOTAL_STEPS_STAGE1 = 2000
VALIDATE_EVERY_STAGE1 = 200
TOTAL_STEPS_STAGE2 = 200
VALIDATE_EVERY_STAGE2 = 10
LR_TRANSFORMER = 5e-6
LR_HEAD = 1e-3
PRETRAINED_TOKENIZER= 'jplu/tf-xlm-roberta-large'
PRETRAINED_MODEL = '/kaggle/input/jigsaw-mlm-finetuned-xlm-r-large'
D = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'
D_TRANS = '/kaggle/input/jigsaw-train-multilingual-coments-google-api/'
print(tf.__version__)
 logging.getLogger().setLevel(logging.NOTSET)
 AUTO = tf.data.experimental.AUTOTUNE
def catboost_model() :
print("Load numerical features")
 df_num = pd.concat(
 [
pd.read_feather(".. /working/feature_num_features.ftr"),
pd.read_feather(".. /working/feature_arithmetic_combi2.ftr"),
],
axis=1, 
)
 y_train = df_num["target"].dropna()
df_num.drop(["target"], axis=1, inplace=True)
 print("Load categorical features")
 df = pd.concat(
 [
pd.read_feather(".. /working/feature_1way_label_encoding.ftr"),
pd.read_feather(".. /working/feature_2way_label_encoding.ftr"),
pd.read_feather(".. /working/feature_3way_including_v22_label_encoding.ftr"),
pd.read_feather(".. /working/feature_round_num_label_encoding.ftr"),
pd.read_feather(".. /working/feature_11way_including_v22_label_encoding.ftr"),
],
axis=1, 
)
 cat_cols = df.columns.tolist()
df = pd.concat([df, df_num], axis=1)
 print("Fit")
 params = {
"loss_function": "Logloss",
"eval_metric": "Logloss",
"learning_rate": 0.03,
"iterations": 3000,
"l2_leaf_reg": 3,
"random_seed": 432013,
"subsample": 0.66,
"od_type": "Iter",
"rsm": 0.2,
"depth": 6,
"border_count": 128,
}
model = cat.CatBoostClassifier(**params)
 train_data = cat.Pool(
 df.iloc[: y_train.shape[0]], label=y_train, cat_features=cat_cols 
)
 fit_model = model.fit(train_data, verbose=30)
 y_pred = fit_model.predict_proba(df.iloc[y_train.shape[0] :])
 submission = pd.read_csv(".. /input/bnp-paribas-cardif-claims-management/sample_submission.csv.zip")
 submission.loc[:, "PredictedProb"] = y_pred[:, 1]
submission.to_csv(".. /working/solution_small_1k.csv", index=False)
 catboost_model()<load_from_csv>
def connect_to_TPU() :

 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
global_batch_size = BATCH_SIZE * strategy.num_replicas_in_sync
return tpu, strategy, global_batch_size
tpu, strategy, global_batch_size = connect_to_TPU()
print("REPLICAS: ", strategy.num_replicas_in_sync )
print('Load data...')
 train = pd.read_csv(".. /input/train.csv")
 target = train['target'].values
train = train.drop(['ID','target','v8','v23','v25','v31','v36','v37','v46','v51','v53','v54','v63','v73','v75','v79','v81','v82','v89','v92','v95','v105','v108','v109','v110','v116','v117','v118','v119','v123','v124','v128'],axis=1)
 test = pd.read_csv(".. /input/test.csv")
 id_test = test['ID'].values
test = test.drop(['ID','v8','v23','v25','v31','v36','v37','v46','v51','v53','v54','v63','v73','v75','v79','v81','v82','v89','v92','v95','v105','v108','v109','v110','v116','v117','v118','v119','v123','v124','v128'],axis=1)
 print('Clearing...')
 for(train_name, train_series),(test_name, test_series)in zip(train.iteritems() ,test.iteritems()):
if train_series.dtype == 'O':
train[train_name], tmp_indexer = pd.factorize(train[train_name])
 test[test_name] = tmp_indexer.get_indexer(test[test_name])
 else:
tmp_len = len(train[train_series.isnull() ])
 if tmp_len>0:
train.loc[train_series.isnull() , train_name] = -999
tmp_len = len(test[test_series.isnull() ])
 if tmp_len>0:
test.loc[test_series.isnull() , test_name] = -999
X_train = train
X_test = test
print('Training...')
 extc = ExtraTreesClassifier(n_estimators=850,max_features= 60,criterion= 'entropy',min_samples_split= 4,
max_depth= 40, min_samples_leaf= 2, n_jobs = -1)
 extc.fit(X_train,target)
 print('Predict...')
 y_pred = extc.predict_proba(X_test)
 pd.DataFrame({"ID": id_test, "PredictedProb": y_pred[:,1]} ).to_csv('extra_trees.csv',index=False )<set_options>
def load_jigsaw_trans(langs=['tr','it','es','ru','fr','pt'],
columns=['comment_text', 'toxic']):
train_6langs=[]
for i in range(len(langs)) :
fn = D_TRANS+'jigsaw-toxic-comment-train-google-%s-cleaned.csv'%langs[i]
train_6langs.append(downsample(pd.read_csv(fn)[columns]))
return train_6langs
def downsample(df):

 ds_df= pd.concat([
df.query('toxic==1'),
df.query('toxic==0' ).sample(sum(df.toxic))
])
 return ds_df
train_df = pd.concat(load_jigsaw_trans())
 val_df = pd.read_csv(D+'validation.csv')
 test_df = pd.read_csv(D+'test.csv')
 sub_df = pd.read_csv(D+'sample_submission.csv' )
gc.enable()<define_variables>
%%time
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
 tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_TOKENIZER)
 X_train = regular_encode(train_df.comment_text.values, tokenizer, maxlen=MAX_LEN)
 X_val = regular_encode(val_df.comment_text.values, tokenizer, maxlen=MAX_LEN)
 X_test = regular_encode(test_df.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train_df.toxic.values.reshape(-1,1)
 y_val = val_df.toxic.values.reshape(-1,1 )
PATH = '/kaggle/input/instacart-market-basket-analysis/'<load_from_csv>
def create_dist_dataset(X, y=None, training=False):
dataset = tf.data.Dataset.from_tensor_slices(X)
 if y is not None:
dataset_y = tf.data.Dataset.from_tensor_slices(y)
 dataset = tf.data.Dataset.zip(( dataset, dataset_y))
if training:
dataset = dataset.shuffle(len(X)).repeat()
dataset = dataset.batch(global_batch_size ).prefetch(AUTO)
 dist_dataset = strategy.experimental_distribute_dataset(dataset)
 return dist_dataset
train_dist_dataset = create_dist_dataset(X_train, y_train, True)
 val_dist_dataset = create_dist_dataset(X_val)
 test_dist_dataset = create_dist_dataset(X_test )
print('loading files...')
 order_products_prior = pd.read_csv(PATH + 'order_products__prior.csv')
 order_products_train = pd.read_csv(PATH + 'order_products__train.csv')
 orders = pd.read_csv(PATH + 'orders.csv')
 products = pd.read_csv(PATH + 'products.csv', usecols=['product_id', 'aisle_id', 'department_id'])
 orders.eval_set = orders.eval_set.replace({'prior': 0, 'train': 1, 'test':2})
 orders.days_since_prior_order = orders.days_since_prior_order.fillna(30)
 print('done loading' )<merge>
%%time
def create_model_and_optimizer() :
with strategy.scope() :
transformer_layer = TFRobertaModel.from_pretrained(PRETRAINED_MODEL)
 model = build_model(transformer_layer)
 optimizer_transformer = Adam(learning_rate=LR_TRANSFORMER)
 optimizer_head = Adam(learning_rate=LR_HEAD)
 return model, optimizer_transformer, optimizer_head
def build_model(transformer):
inp = Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_word_ids")
 x = transformer(inp)[0][:, 0, :]
x = Dropout(DROPOUT )(x)
 out = Dense(1, activation='sigmoid', name='custom_head' )(x)
 model = Model(inputs=[inp], outputs=[out])
 return model
model, optimizer_transformer, optimizer_head = create_model_and_optimizer()
model.summary()
prior_orders = pd.merge(orders, order_products_prior, on='order_id', how='inner')
 prior_orders.head()<drop_column>
def define_losses_and_metrics() :
with strategy.scope() :
loss_object = tf.keras.losses.BinaryCrossentropy(
 reduction=tf.keras.losses.Reduction.NONE, from_logits=False)
 def compute_loss(labels, predictions):
per_example_loss = loss_object(labels, predictions)
 loss = tf.nn.compute_average_loss(
 per_example_loss, global_batch_size = global_batch_size)
 return loss
train_accuracy_metric = tf.keras.metrics.AUC(name='training_AUC')
 return compute_loss, train_accuracy_metric
def train(train_dist_dataset, val_dist_dataset=None, y_val=None,
total_steps=2000, validate_every=200):
best_weights, history = None, []
step = 0
for tensor in train_dist_dataset:
distributed_train_step(tensor)
 step+=1
if(step % validate_every == 0):
train_metric = train_accuracy_metric.result().numpy()
print("Step %d, train AUC: %.5f" %(step, train_metric))
if val_dist_dataset:
val_metric = roc_auc_score(y_val, predict(val_dist_dataset))
print("Step %d, val AUC: %.5f" %(step,val_metric))
history.append(val_metric)
 if history[-1] == max(history):
best_weights = model.get_weights()
train_accuracy_metric.reset_states()
if step == total_steps:
break
model.set_weights(best_weights)
 @tf.function
def distributed_train_step(data):
strategy.experimental_run_v2(train_step, args=(data,))
def train_step(inputs):
features, labels = inputs
transformer_trainable_variables = [ v for v in model.trainable_variables
if(( 'pooler' not in v.name)and 
('custom' not in v.name)) ]
head_trainable_variables = [ v for v in model.trainable_variables
if 'custom' in v.name]
with tf.GradientTape(persistent=True)as tape:
predictions = model(features, training=True)
 loss = compute_loss(labels, predictions)
 gradients_transformer = tape.gradient(loss, transformer_trainable_variables)
 gradients_head = tape.gradient(loss, head_trainable_variables)
 del tape
optimizer_transformer.apply_gradients(zip(gradients_transformer,
transformer_trainable_variables))
optimizer_head.apply_gradients(zip(gradients_head,
head_trainable_variables))
train_accuracy_metric.update_state(labels, predictions)
 def predict(dataset):
predictions = []
for tensor in dataset:
predictions.append(distributed_prediction_step(tensor))
predictions = np.vstack(list(map(np.vstack,predictions)))
 return predictions
@tf.function
def distributed_prediction_step(data):
predictions = strategy.experimental_run_v2(prediction_step, args=(data,))
return strategy.experimental_local_results(predictions)
 def prediction_step(inputs):
features = inputs
predictions = model(features, training=False)
 return predictions
compute_loss, train_accuracy_metric = define_losses_and_metrics()
del order_products_prior
gc.collect()<data_type_conversions>
%%time
train(train_dist_dataset, val_dist_dataset, y_val,
TOTAL_STEPS_STAGE1, VALIDATE_EVERY_STAGE1 )
users = prior_orders.groupby(by='user_id')['order_number'].aggregate('max' ).to_frame('u_num_of_orders' ).reset_index()
users.head()<create_dataframe>
%%time
optimizer_head.learning_rate.assign(1e-4)
 X_train, X_val, y_train, y_val = train_test_split(X_val, y_val, test_size = 0.1)
 train_dist_dataset = create_dist_dataset(X_train, y_train, training=True)
 val_dist_dataset = create_dist_dataset(X_val, y_val)
 train(train_dist_dataset, val_dist_dataset, y_val,
total_steps = TOTAL_STEPS_STAGE2,
validate_every = VALIDATE_EVERY_STAGE2 )
total_prd_per_order = prior_orders.groupby(by=['user_id', 'order_id'])['product_id'].aggregate('count' ).to_frame('total_products_per_order' ).reset_index()
avg_products = total_prd_per_order.groupby(by=['user_id'])['total_products_per_order'].mean().to_frame('u_avg_prd' ).reset_index()
avg_products.head()
del [total_prd_per_order]
gc.collect()
avg_products.head() 
<data_type_conversions>
%%time
sub_df['toxic'] = predict(test_dist_dataset)[:,0]
sub_df.to_csv('submission.csv', index=False )