Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
data_test = data[data.eval_set == 2]
data_test.head()<drop_column>
def seed_everything(seed=0):
np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
SEED = 36
seed_everything(SEED)
data_test = data_test.set_index(['user_id', 'product_id'])
 data_test.drop(['eval_set', 'order_id'], axis=1, inplace=True )<drop_column>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=True,
return_token_type_ids=True,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'],dtype=np.int32),np.array(enc_di['attention_mask'],dtype=np.int32),np.array(enc_di['token_type_ids'],dtype=np.int32 )
del [data, orders_future, products, order_products_train]
gc.collect()<import_modules>
def focal_loss(gamma=2., alpha=.2):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed
from sklearn.model_selection import train_test_split, GridSearchCV, KFold
from sklearn.metrics import f1_score, classification_report
from scikitplot.metrics import plot_confusion_matrix
from scikitplot.classifiers import plot_feature_importances
import xgboost as xgb
import lightgbm as lgb<split>
def build_model(transformer,max_len=512):
input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 sequence_output = transformer(( input_ids,input_mask,segment_ids)) [0]
gp = GlobalMaxPooling1D()(sequence_output)
 ap = GlobalAveragePooling1D()(sequence_output)
 stack = concatenate([gp,ap],axis=1)
 out = Dense(1, activation='sigmoid' )(stack)
 model = Model(inputs=[input_ids,input_mask,segment_ids], outputs=out)
 model.compile(Adam(lr=0.25e-5), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.AUC() ])
 return model
X = data_train.drop('reordered', axis=1)
 y = data_train.reordered
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=10 )<train_model>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
lgbm.fit(X, y )<predict_on_test>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 2
BATCH_SIZE = 16* strategy.num_replicas_in_sync
MAX_LEN = 256
MODEL = 'jplu/tf-xlm-roberta-large'
y_pred_test =(lgbm.predict_proba(data_test)[:, 1] >= 0.21 ).astype('int' )<prepare_output>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
data_test['prediction'] = y_pred_test
data_test.head()<load_from_csv>
valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
orders = pd.read_csv(PATH + 'orders.csv')
 orders_test = orders.loc[orders.eval_set == 'test', ['user_id', 'order_id']]
orders_test.head()<merge>
df_train = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv")
 df_train['lang'] = 'lang'
df_train = df_train[['comment_text', 'toxic', 'lang']].head(0)
final = final.merge(orders_test, on='user_id', how='left')
 final.head()<drop_column>
for lang in ['es', 'tr', 'it', 'ru', 'pt', 'fr']:
df_lang = pd.read_csv(f'.. /input/jigsawtanslatedgoogle/jigsaw-unintended-bias-train_{lang}_clean.csv')[['id', 'comment_text', 'toxic']]
df_lang = df_lang[(df_lang['toxic'] < 0.2)|(df_lang['toxic'] > 0.5)]
df_lang['toxic'] = df_lang['toxic'].apply(lambda x:round(x)).astype(np.int32)
 df_lang = df_lang[~df_lang['comment_text'].isna() ]
df_lang_sampled = pd.concat([
df_lang[['comment_text', 'toxic']].query('toxic==1' ).sample(n=35000),
df_lang[['comment_text', 'toxic']].query('toxic==0' ).sample(n=80000),])
 del df_lang
df_lang_sampled['lang'] = lang
df_train = df_train.append(df_lang_sampled)
final = final.drop('user_id', axis=1 )<data_type_conversions>
def text_head_tail(input_df, text_col='comment_text'):
df = input_df.copy()
df['text_head'] = df[text_col].apply(lambda x:' '.join(x.split() [:50]))
df['text_tail'] = df[text_col].apply(lambda x:' '.join(x.split() [-70:]))
df['text_length'] = df[text_col].apply(lambda x:len(x.split()))
df[text_col] = np.where(df['text_length']>120,
df['text_head'] + ' '+df['text_tail'],
df[text_col])
 df['text_length_2'] = df[text_col].apply(lambda x:len(x.split()))
return df
df_train = text_head_tail(df_train)
 valid = text_head_tail(valid)
 test = text_head_tail(test, 'content')
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()<feature_engineering>
x_train = regular_encode(df_train['comment_text'].values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid['comment_text'].values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test['content'].values, tokenizer, maxlen=MAX_LEN)
 y_train = df_train.toxic.values
y_valid = valid.toxic.values
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect() 
<prepare_output>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.shuffle(len(y_train)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
 )
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()<save_to_csv>
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
sub.to_csv('sub.csv', index=False, header=True )<set_options>
n_steps = df_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)
gc.collect()<set_options>
n_steps = valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=4,
shuffle=False,
)
<load_from_csv><EOS> 
sub['toxic'] = model.predict(x_test, verbose=1)
 sub.to_csv('submission.csv', index=False)
 model.save_weights('checkpoint.h5', overwrite=True)
 sub.head(55 )