Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<train_model>
import os
import re
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input, Embedding, Conv1D, MaxPooling1D, Flatten
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import transformers
from transformers import TFAutoModel, AutoTokenizer, RobertaTokenizerFast
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
%%time
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = linear_model.LinearRegression(n_jobs=-1)
 lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
size = 10_000
train = pd.read_csv('.. /input/train.csv', iterator=True, chunksize=size, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})
 strain = []
group = []
for df in train:
group.append(df)
 group = group[-int(150_000/size):]
df2 = pd.concat(group)
 if len(df2)==150_000:
seismic_chunk = df2['acoustic_data']
fullmean = seismic_chunk.mean()
fullstd = seismic_chunk.std()
fullmax = seismic_chunk.max()
fullmin = seismic_chunk.min()
fullmad = seismic_chunk.mad()
fullkurtosis = seismic_chunk.kurtosis()
fullskew = seismic_chunk.skew()
sc_abs = np.abs(seismic_chunk)
 q01_abs = np.quantile(sc_abs,0.01)
 q05_abs = np.quantile(sc_abs,0.05)
 q95_abs = np.quantile(sc_abs,0.95)
 q99_abs = np.quantile(sc_abs,0.99)
 roll = seismic_chunk.rolling(100 ).std().dropna().values
q01_roll = np.quantile(roll,0.01)
 q05_roll = np.quantile(roll,0.05)
 q95_roll = np.quantile(roll,0.95)
 q99_roll = np.quantile(roll,0.99)
 seismic_chunk = df2['acoustic_data'].values
q01 = np.quantile(seismic_chunk,0.01)
 q05 = np.quantile(seismic_chunk,0.05)
 q95 = np.quantile(seismic_chunk,0.95)
 q99 = np.quantile(seismic_chunk,0.99)
 abs_max = np.abs(seismic_chunk ).max()
abs_mean = np.abs(seismic_chunk ).mean()
abs_std = np.abs(seismic_chunk ).std()
trend = add_trend_feature(seismic_chunk)
 abs_trend = add_trend_feature(seismic_chunk, abs_values=True)
 lastTTF = df2['time_to_failure'].values[-1]
strain.append([fullmean, fullstd, fullmax, fullmin, fullmad, fullkurtosis, fullskew, q01, q05, q95, q99, q01_abs, q05_abs, q95_abs, q99_abs, q01_roll, q05_roll, q95_roll, q99_roll, abs_max, abs_mean, abs_std, trend, abs_trend, lastTTF])
 strain = pd.DataFrame(strain, columns=['fullmean', 'fullstd', 'fullmax', 'fullmin', 'fullmad', 'fullkurtosis', 'fullskew', 'q01', 'q05', 'q95', 'q99', 'q01_abs', 'q05_abs', 'q95_abs', 'q99_abs', 'q01_roll', 'q05_roll', 'q95_roll', 'q99_roll', 'abs_max', 'abs_mean', 'abs_std', 'trend', 'abs_trend', 'time_to_failure'])
 test = glob.glob('.. /input/test/**')
 stest = []
for path in test:
df = pd.read_csv(path, dtype={'acoustic_data': np.int16})
 seg_id = path.split('/')[-1].split('.')[0]
seismic_chunk = df['acoustic_data']
fullmean = seismic_chunk.mean()
fullstd = seismic_chunk.std()
fullmax = seismic_chunk.max()
fullmin = seismic_chunk.min()
fullmad = seismic_chunk.mad()
fullkurtosis = seismic_chunk.kurtosis()
fullskew = seismic_chunk.skew()
sc_abs = np.abs(seismic_chunk)
 q01_abs = np.quantile(sc_abs,0.01)
 q05_abs = np.quantile(sc_abs,0.05)
 q95_abs = np.quantile(sc_abs,0.95)
 q99_abs = np.quantile(sc_abs,0.99)
 roll = seismic_chunk.rolling(100 ).std().dropna().values
q01_roll = np.quantile(roll,0.01)
 q05_roll = np.quantile(roll,0.05)
 q95_roll = np.quantile(roll,0.95)
 q99_roll = np.quantile(roll,0.99)
 seismic_chunk = df['acoustic_data'].values
q01 = np.quantile(seismic_chunk,0.01)
 q05 = np.quantile(seismic_chunk,0.05)
 q95 = np.quantile(seismic_chunk,0.95)
 q99 = np.quantile(seismic_chunk,0.99)
 abs_max = np.abs(seismic_chunk ).max()
abs_mean = np.abs(seismic_chunk ).mean()
abs_std = np.abs(seismic_chunk ).std()
trend = add_trend_feature(seismic_chunk)
 abs_trend = add_trend_feature(seismic_chunk, abs_values=True)
 stest.append([seg_id, fullmean, fullstd, fullmax, fullmin, fullmad, fullkurtosis, fullskew, q01, q05, q95, q99, q01_abs, q05_abs, q95_abs, q99_abs, q01_roll, q05_roll, q95_roll, q99_roll, abs_max, abs_mean, abs_std, trend, abs_trend])
 stest = pd.DataFrame(stest, columns=['seg_id', 'fullmean', 'fullstd', 'fullmax', 'fullmin', 'fullmad', 'fullkurtosis', 'fullskew', 'q01', 'q05', 'q95', 'q99', 'q01_abs', 'q05_abs', 'q95_abs', 'q99_abs', 'q01_roll', 'q05_roll', 'q95_roll', 'q99_roll', 'abs_max', 'abs_mean', 'abs_std', 'trend', 'abs_trend'])
 sub = pd.read_csv('.. /input/sample_submission.csv')
 print(strain.shape, stest.shape, sub.shape )<save_to_csv>
def clean_sentence(sentence):

 clean_sent =
for word in sentence.split() :
if len(word)> 1 and not re.match(r'.*[0-9]+', word):
clean_sent = "{} {}".format(clean_sent, word)
 clean_sent = re.sub(r'[.,:;?!/\|@
clean_sent = text = re.sub(r'(http|www)\S*', '', clean_sent)
 return clean_sent.strip()
def clean_df(df, column):

 df = df.apply(lambda x: x.astype(str ).str.lower())
 df[column] = df[column].apply(lambda x: clean_sentence(x))
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
 def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def build_CNN_model() :
input_layer = Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_words_ids")
 embedding = Embedding(VOCAB_SIZE[0], VOCAB_SIZE[1], input_length=MAX_LEN, name='embed' )(input_layer)
 conv_1 = Conv1D(256,(100), activation='relu' )(embedding)
 max_pool = MaxPooling1D()(conv_1)
 conv_1 = Conv1D(128,(5), activation='relu' )(max_pool)
 max_pool = MaxPooling1D()(conv_1)
 dense = Dense(128, activation='relu' )(max_pool)
 dense = Dense(128, activation='relu' )(dense)
 flatten = Flatten()(dense)
 out = Dense(1, activation='sigmoid' )(flatten)
 model = Model(inputs = input_layer, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
%%time
col = [c for c in strain.columns if c not in ['time_to_failure']]
scaler = preprocessing.StandardScaler()
scaled_train = scaler.fit_transform(strain[col])
 scaled_test = scaler.transform(stest[col])
 m = CatBoostRegressor(loss_function='MAE')
 m.fit(scaled_train, strain['time_to_failure'], silent=True)
 print(metrics.mean_absolute_error(strain['time_to_failure'], m.predict(scaled_train)))
 stest['time_to_failure'] = m.predict(scaled_test)
 stest[['seg_id','time_to_failure']].to_csv('submission.csv', index=False )<import_modules>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
!pip install fastprogress==0.2.3
!pip install fastai==1.0.61 
<set_options>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 1
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
device = 'cuda' if torch.cuda.is_available() else 'cpu'<define_variables>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
path = Path('/kaggle/input/plant-pathology-2020-fgvc7/' )<load_from_csv>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train3 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv")
 train4 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-fr-cleaned.csv")
 train5 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-it-cleaned.csv")
 train6 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-pt-cleaned.csv")
 train7 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-ru-cleaned.csv")
 train8 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv")
 valid1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 valid2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv')
 test1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 test2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
train_df = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'test.csv')
 sample_df = pd.read_csv(path/'sample_submission.csv' )<feature_engineering>
clean_df(train, 'comment_text')
 train.toxic = train.toxic.round().astype(int)
 train = train.sample(frac = 1)
 clean_df(valid, 'comment_text')
 clean_df(test1, 'content')
 clean_df(test2, 'content' )
test_df['image_id'] = test_df['image_id'] + '.jpg'
train_df['image_id'] = train_df['image_id'] + '.jpg'<define_variables>
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test1 = regular_encode(test1.content.values, tokenizer, maxlen=MAX_LEN)
 x_test2 = regular_encode(test2.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
def get_label(row):
if row.healthy == 1:
return 'healthy'
elif row.rust == 1:
return 'rust'
elif row.scab == 1:
return 'scab'
else:
return 'multiple_diseases'<feature_engineering>
x_train1 = regular_encode(train1.comment_text.values, tokenizer, maxlen=MAX_LEN)
 train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
train_df['label'] = train_df.apply(get_label, axis=1 )<drop_column>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test1_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test1)
.batch(BATCH_SIZE)
)
 test2_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test2)
.batch(BATCH_SIZE)
 )
train_df = train_df[['image_id', 'label']]<count_values>
VOCAB_SIZE = x_train.shape
c = Counter(train_df.label), len(train_df)
 c<categorify>
with strategy.scope() :
cnn_model = build_CNN_model()
cnn_model.summary()
def get_img_id_from_idx(idx):
return train_df.iloc[idx].image_id<load_pretrained>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = roberta_model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
img_size = 1024
train_data = create_databunch_from_img_size(img_size, 32 )<categorify>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = roberta_model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
class ROCAUCScore(Callback):
def on_epoch_begin(self, **kwargs):
self.targets, self.preds = [], []
def on_batch_end(self, last_output, last_target, **kwargs):
self.targets.extend(last_target.tolist())
 self.preds.extend(list(map(scipy.special.softmax, last_output.tolist())))
 def on_epoch_end(self, last_metrics, **kwargs):
sc = roc_auc_score(self.targets, self.preds, multi_class='ovr')
 return add_metrics(last_metrics, sc )<train_model>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = cnn_model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS*5
)
class StopTrainingAtEpoch(Callback):
def __init__(self, learn, n):
self.learn = learn
self.n = n
def on_epoch_end(self, epoch, **kwargs):
if epoch == self.n:
self.learn.save('model-' + str(self.n))
return {'stop_training': True}<find_best_params>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = cnn_model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
ce_weight = torch.tensor([1.0, 4.0, 1.0, 1.0] ).to(device)
 print(ce_weight)
 ce_weight=None
learn = cnn_learner(train_data, models.alexnet, metrics=[ROCAUCScore() , accuracy, error_rate, Precision(average='macro'), Recall(average='macro'), FBeta(beta=1.0, average='macro')],
loss_func=CrossEntropyFlat(reduction='mean', weight=ce_weight), ps=[0.5, 0.5, 0.5], wd=0.01,
path='/kaggle/working', callback_fns=[ShowGraph])
<load_pretrained>
multi_ling_sub = roberta_model.predict(test1_dataset, verbose=1)
 eng_sub = roberta_model.predict(test2_dataset, verbose=1)
 sub['toxic'] = multi_ling_sub*0.5 + eng_sub*0.5
learn.load('/kaggle/input/plant-pathology-fgvc7-fastai/models/stage-06');
learn.freeze()<load_pretrained>
sub.to_csv('submission.csv', index=False )
<save_model><EOS> 
sub.to_csv('submission.csv', index=False )