Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
combined_df['Min_Weeks'] = combined_df['Weeks']
combined_df.loc[combined_df.data_type == 'Test','Min_Weeks'] = np.nan
combined_df['Min_Weeks'] = combined_df.groupby('Patient')['Min_Weeks'].transform('min' )<remove_duplicates>
path="/kaggle/input/jigsaw-multilingual-toxic-comment-classification/"
base = combined_df.loc[combined_df.Weeks == combined_df.Min_Weeks]
base = base[['Patient','FVC']].rename(columns = {'FVC':'min_FVC'})
 base.drop_duplicates(keep = 'first', inplace = True, subset = ['Patient'] )<data_type_conversions>
training_set=pd.read_csv(path+"jigsaw-toxic-comment-train.csv")
 test_set=pd.read_csv(path+"test.csv")
 validation_set=pd.read_csv(path+"validation.csv")
 sub=pd.read_csv(path+"sample_submission.csv" )
combined_df.Weeks = combined_df.Weeks.astype(int)
 combined_df.Min_Weeks = combined_df.Min_Weeks.astype(float )<merge>
training_set.isnull().sum()
combined_df = combined_df.merge(base, on='Patient', how='left')
 combined_df['Deviation_Weeks'] = combined_df['Weeks'] - combined_df['Min_Weeks']
del base<categorify>
test_set.isnull().sum()
combined_df = pd.concat([combined_df, pd.get_dummies(combined_df[['Sex','SmokingStatus']])], axis = 1 )<normalization>
validation_set.isnull().sum()
scaler = MinMaxScaler()
scaled = pd.DataFrame(scaler.fit_transform(combined_df[['Age','Percent','min_FVC','Deviation_Weeks']]),
columns = ['scaled_Age', 'scaled_Percent', 'scaled_FVC', 'scaled_Deviation_Weeks'])
 combined_df = pd.concat([combined_df, scaled], axis = 1 )<define_variables>
training_set.drop(["severe_toxic","obscene","threat","insult","identity_hate","id"],axis=1,inplace=True)
 training_set.head()
feature_columns = ['Sex_Male','Sex_Female','SmokingStatus_Ex-smoker','SmokingStatus_Never smoked','SmokingStatus_Currently smokes',
'scaled_Age','scaled_Percent','scaled_Deviation_Weeks','scaled_FVC']<filter>
test_set.drop(["lang","id"],axis=1,inplace=True)
 validation_set.drop(["lang","id"],axis=1,inplace=True )
train_df = combined_df.loc[combined_df.data_type == 'Train']
test_df = combined_df.loc[combined_df.data_type == 'Val']
submission_df = combined_df.loc[combined_df.data_type == 'Test']
del combined_df<compute_test_metric>
def clean_text(text):
text=text.lower()
text=re.sub("
"," ",text)
 text=re.sub("\[\[User.*"," ",text)
 text=re.sub("\(http://.*?\s\(http://.*\)"," ",text)
 text=re.sub("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"," ",text)
 text=re.sub("[.,]"," ",text)
 text=text.split()
text=" ".join(text)
 return text
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
x1 = Input(( 9,), name="Patient")
 x2 = Dense(100, activation="relu", name="d1" )(x1)
 x3 = Dense(100, activation="relu", name="d2" )(x2)
 p1 = Dense(3, activation="relu", name="p1" )(x3)
 p2 = Dense(3, activation="relu", name="p2" )(x3)
 preds = Lambda(lambda x3: x3[0] + tf.cumsum(x3[1], axis=1),
name="preds" )([p1, p2])
 model = Model(x1, preds, name="CNN")
 model.compile(loss = mloss(0.8), optimizer = tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.005,
amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
training_set["comment_text"]=training_set["comment_text"].apply(str ).apply(lambda x:clean_text(x))
test_set["content"]=test_set["content"].apply(str ).apply(lambda x:clean_text(x))
validation_set["comment_text"]=validation_set["comment_text"].apply(str ).apply(lambda x:clean_text(x))
y = train_df['FVC'].values
z = train_df[feature_columns].values
sub = submission_df[feature_columns].values
pe = np.zeros(( sub.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
from keras.models import Sequential
from keras.layers import Embedding
from keras.layers import Dense
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Flatten
from keras.layers import SimpleRNN
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
NFOLD = 5
BATCH_SIZE=128
kf = KFold(n_splits=NFOLD )<predict_on_test>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 model = make_model()
model.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", model.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", model.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = model.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += model.predict(sub, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<feature_engineering>
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
submission_df['FVC1'] = pe[:, 1]
submission_df['Confidence1'] = pe[:, 2] - pe[:, 0]<create_dataframe>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 256
MODEL = 'jplu/tf-xlm-roberta-large'
subm = submission_df[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
x_train = regular_encode(training_set.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(validation_set.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test_set.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = training_set.toxic.values
y_valid = validation_set.toxic.values
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
 )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<install_modules>
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index<set_options>
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<set_options>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )<load_from_csv>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
<concatenate><EOS> 
sub['toxic'] = model.predict(test_dataset)
 sub.head()
sub.to_csv('submission.csv', index=False )