Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<prepare_x_and_y>
x_train = preprocess_df(train_df.drop(columns="sig_id"))
x_test =preprocess_df(test_df.drop(columns="sig_id"))
y_train = train_target_df.drop(columns="sig_id")
 N_FEATURES = x_train.shape[1]<data_type_conversions>
%matplotlib inline
x_train = x_train.astype({'cp_time':int})
 x_test = x_test.astype({'cp_time':int} )<choose_model_class>
sample_submission = pd.read_csv('/kaggle/input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv.zip')
 test_labels = pd.read_csv('//kaggle/input/jigsaw-toxic-comment-classification-challenge/test_labels.csv.zip')
 train = pd.read_csv('//kaggle/input/jigsaw-toxic-comment-classification-challenge/train.csv.zip')
 test = pd.read_csv('//kaggle/input/jigsaw-toxic-comment-classification-challenge/test.csv.zip' )
def create_model() :
model = tf.keras.Sequential([
tf.keras.layers.Input(N_FEATURES),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(N_TARGETS, activation="sigmoid"))
])
 model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate = LR), loss='binary_crossentropy', metrics=["accuracy"])
 return model<train_model>
from sklearn.model_selection import train_test_split
def build_train(resume_models = None, repeat_number = 0, folds = 5, skip_folds = 0):
models = []
oof_preds = y_train.copy()
kfold = KFold(folds, shuffle = True)
 for fold,(train_ind, val_ind)in enumerate(kfold.split(x_train)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'binary_crossentropy', factor = 0.4, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'auto')
 checkpoint_path = f'repeat:{repeat_number}_Fold:{fold}.hdf5'
cb_checkpt = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min')
 model = create_model()
model.fit(x_train.values[train_ind],
y_train.values[train_ind],
validation_data=(x_train.values[val_ind], y_train.values[val_ind]),
callbacks = [cb_lr_schedule, cb_checkpt],
epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=2 
)
 model.load_weights(checkpoint_path)
 oof_preds.loc[val_ind, :] = model.predict(x_train.values[val_ind])
 models.append(model)
 print('train:')
 print(list(zip(model.metrics_names, model.evaluate(x_train.values[train_ind], y_train.values[train_ind], verbose=0, batch_size=32))))
print('val:')
 print(list(zip(model.metrics_names, model.evaluate(x_train.values[val_ind], y_train.values[val_ind], verbose=0, batch_size=32))))
return models, oof_preds<train_model>
from sklearn.model_selection import train_test_split
start = time.time()
models = []
oof_preds = []
seed_everything(SEED)
 for i in range(REPEATS):
m, oof = build_train(repeat_number = i, folds=FOLDS)
 models = models + m
oof_preds.append(oof)
 finish = time.time() -start
print(finish )<predict_on_test>
X = train.comment_text.values
y = train[['toxic','severe_toxic','obscene','threat','insult','identity_hate']].values
X_train, X_val, y_train, y_val =train_test_split(X, y, test_size=0.1, random_state=2020 )
models[1].predict(x_test )<save_to_csv>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print(f'There are {torch.cuda.device_count() } GPU(s)available.')
 print('Device name:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
test_preds = sub.copy()
test_preds[target_cols] = 0
for model in models:
test_preds.loc[:,target_cols] += model.predict(x_test)
 test_preds.loc[:,target_cols] /= len(models)
 test_preds.loc[x_test['cp_type'] == 0, target_cols] = 0
test_preds.to_csv('submission.csv', index=False )<install_modules>
def text_preprocessing(text):

 text = re.sub(r'(@.*?)[\s]', ' ', text)
 text = re.sub(r'&amp;', '&', text)
 text = re.sub(r'\s+', ' ', text ).strip()
return text
sys.path.append(".. /input/keras-one-cycle-lr/")
 sys.path.append(".. /input/keras-one-cycle-lr/one_cycle_lr")
 !pip install.. /input/keras-one-cycle-lr<set_options>
print('Original:
', X[0])
 print('
Processed: ', text_preprocessing(X[0]))
sys.path.append('.. /input/moa-scripts')
 warnings.simplefilter('ignore')
 def seed_everything(seed=42):
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(seed)
 random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything() 
<set_options>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',do_lower_case=True)
 def preprocessing_for_bert(data):

 input_ids = []
attention_masks = []
for sent in data:
encoded_sent = tokenizer.encode_plus(
 text = text_preprocessing(sent),
add_special_tokens = True,
max_length= MAX_LEN ,
pad_to_max_length = True,
return_attention_mask= True 
)
 input_ids.append(encoded_sent.get('input_ids'))
attention_masks.append(encoded_sent.get('attention_mask'))
input_ids = torch.tensor(input_ids)
 attention_masks = torch.tensor(attention_masks)
 return input_ids,attention_masks
tf.keras.backend.clear_session()
tf.config.optimizer.set_jit(True)
<split>
all_text = np.concatenate([train.comment_text.values,test.comment_text.values])
 len_sent = [len(text_preprocessing(sent)) for sent in all_text]
avg_len = np.mean(len_sent)
 print('Avg length: ',avg_len )
train, target, y_nonscored, test, submission = load_datasets(".. /input/lish-moa")
 X, y, test, test_control = preprocess(train, target, test, standard=False, onehot=True)
 X, y, X_holdout, y_holdout, split_index, index, classnames, features = split(X, y, drop_rare_labels=False)
 genes = [i for i,col in enumerate(features)if col.startswith('g-')]
cells = [i for i,col in enumerate(features)if col.startswith('c-')]<concatenate>
MAX_LEN = 150
token_ids = list(preprocessing_for_bert([X[0]])[0].squeeze().numpy())
 print('Original: ',X[0])
 print('Token IDs: ',token_ids )
n = len(X)
 c1, c2 = 64, 16
pca1,pca2 = PCA(c1), PCA(c2)
 pca1_res = pca1.fit_transform(np.concatenate([X, test]))
pca2_res = pca2.fit_transform(np.concatenate([X, test]))
X = np.concatenate([X, pca1_res[:n], pca2_res[:n]], axis=1)
 test = np.concatenate([test, pca1_res[n:], pca2_res[n:]], axis=1)
 features = list(features)
 features += [f'pca_genes_{i}' for i in range(c1)] + [f'pca_cells_{i}' for i in range(c2)]
assert len(features)== X.shape[1] == test.shape[1]<train_model>
print('Tokenizing data...')
 train_inputs, train_masks = preprocessing_for_bert(X_train)
 val_inputs, val_masks = preprocessing_for_bert(X_val )
var = VarianceThreshold(threshold=0.5)
 var.fit(np.concatenate([X, test])[:, 4:])
 mask = np.concatenate([[True]*4, var.get_support() ])
 X = X[:, mask]
test = test[:, mask]<compute_test_metric>
train_labels = torch.tensor(y_train)
 val_labels = torch.tensor(y_val)
 batch_size = 32
train_data = TensorDataset(train_inputs,train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 val_data = TensorDataset(val_inputs, val_masks, val_labels)
 val_sampler = SequentialSampler(val_data)
 val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=batch_size )
def weighted_logloss_keras(w=0.5):
alpha = K.variable(w)
 def loss(y_true, y_pred):
y_pred = K.clip(y_pred, K.epsilon() , 1 - K.epsilon())
 return - 2 * K.mean(( alpha * y_true * K.log(y_pred)+(1 - alpha)*(1 - y_true)*(K.log(1 - y_pred))))
return loss
def logloss_keras(y_true, y_pred):
y_pred = K.clip(y_pred, K.epsilon() , 1 - K.epsilon())
 return - K.mean(y_true * K.log(y_pred)+(1 - y_true)*(K.log(1 - y_pred)))
 max_norm = tf.keras.constraints.max_norm
he_uniform = lambda scale, seed: tf.keras.initializers.VarianceScaling(mode='fan_in', distribution='uniform', scale=scale, seed=seed)
 constant = lambda x: tf.keras.initializers.Constant(value=x)
 l2_reg = lambda x: tf.keras.regularizers.l2(x)
 class DenseBlock(L.Layer):
def __init__(self,
dropout_rate=0.5,
dense=(1024, 6.0, 1e-5, None),
prelu=(0.25, 0.0, None),
clipnorm=None,
seed=42):
super(DenseBlock, self ).__init__()
self.batch_norm = L.BatchNormalization()
self.dropout = L.Dropout(dropout_rate)
 self.dense = L.Dense(
 dense[0], activation=None,
kernel_initializer=he_uniform(dense[1], seed),
kernel_regularizer=l2_reg(dense[2]),
kernel_constraint =max_norm(dense[3])if dense[3] else None 
)
 self.activation = L.ReLU()
self.clipnorm = tf.keras.constraints.MaxNorm(clipnorm)if clipnorm else None
def call(self, inputs):
x = self.batch_norm(inputs)
 x = self.dropout(x)
 x = self.dense(x)
 x = self.activation(x)
 if self.clipnorm:
x = self.clipnorm(x)
 return x
def create_model(num_columns,
dropout=(0.25, 0.5),
hidden=(1024, 1024),
he_scale=6.0,
kernel=(1e-5, None),
activation=(0.25, 0.0, None),
top_layer_bias=1.0,
clipnorm=None,
learning_rate=1e-3,
weight_decay=1e-5,
lookahead_sync=7,
label_smoothing=1e-6,
l2top=1,
seed=42,
metrics=[]):
model = tf.keras.Sequential()
model.add(L.Input(num_columns))
for i in range(len(hidden)) :
d = dropout[0] if i == 0 else dropout[1]
model.add(DenseBlock 
(dropout_rate=d,
dense=(hidden[i], he_scale, kernel[0], kernel[1]),
prelu=activation,
clipnorm=clipnorm,
seed=seed 
)
)
 model.add(L.BatchNormalization())
 model.add(L.Dropout(dropout[1]))
model.add(tfa.layers.WeightNormalization(L.Dense(
 206, activation="sigmoid",
kernel_initializer=he_uniform(he_scale, seed),
bias_initializer=constant(top_layer_bias),
kernel_regularizer=l2_reg(kernel[0])if l2top else None 
)))
 model.compile(
 optimizer=tfa.optimizers.Lookahead(tfa.optimizers.AdamW(learning_rate=learning_rate, weight_decay=weight_decay), sync_period=lookahead_sync),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=metrics 
)
 return model 
<init_hyperparams>
%%time
class BertClassifier(nn.Module):

 def __init__(self, freeze_bert=False):

 super(BertClassifier,self ).__init__()
D_in, H, D_out = 768, 50 , 6
self.bert = BertModel.from_pretrained('bert-base-uncased')
 self.classifier = nn.Sequential(
 nn.Linear(D_in, H),
nn.ReLU() ,
nn.Linear(H, D_out))
if freeze_bert:
for param in self.bert.parameters() :
param.requires_grad = False
def forward(self,input_ids,attention_mask):

 outputs = self.bert(input_ids=input_ids,
attention_mask = attention_mask)
 last_hidden_state_cls = outputs[0][:,0,:]
logits = self.classifier(last_hidden_state_cls)
 return logits
model_params = dict(
 dropout =(0.05, 0.65),
hidden =(1024, 1024),
kernel =(3e-6, None),
activation =(0.05, 6e-6, 4),
he_scale =6,
clipnorm =None,
label_smoothing =0,
weight_decay =1e-5,
l2top =1,
top_layer_bias =1.0,
learning_rate =5e-4,
lookahead_sync =7,
seed=42 
)
 scheduler_params = dict(
 max_lr=1e-2,
pct_start=0.2,
start_div=1e4,
end_div=1e5 
)
 fit_params = dict(
 epochs=25,
batch_size=96,
verbose=0
)<train_model>
def initialize_model(epochs=4):

 bert_classifier = BertClassifier(freeze_bert=False)
 bert_classifier.to(device)
 optimizer = AdamW(bert_classifier.parameters() ,
lr=5e-5,
eps=1e-8 
)
 total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
 return bert_classifier, optimizer, scheduler
%%time
def crossval(X, y, X_test,
model_params, scheduler_params, fit_params,
splits=[(MultilabelStratifiedKFold, 5)],
seeds=[42],
fold_mask=None,
verbose=1,
pseudo=-1 
):
oof = np.zeros(( len(X_test), y.shape[1]), dtype=np.float64)
 preds = np.zeros_like(y, dtype=np.float64)
 history = []
for seed in seeds:
seed_everything(seed)
 model_params['seed'] = seed
for kf, n_splits in splits:
cv = kf(n_splits=n_splits, shuffle=True, random_state=seed)
 for fold,(train_index, valid_index)in enumerate(cv.split(X, y)) :
if fold_mask and fold not in fold_mask:
continue
iter_id = f'{str(kf ).split(".")[-1][:-2]}-seed{seed}-fold{fold+1}'
if verbose: print(iter_id, end=': ')
 X_train, y_train, X_valid, y_valid = X[train_index].copy() , y[train_index].copy() , X[valid_index].copy() , y[valid_index].copy()
model = create_model(X.shape[1], **model_params)
 checkpoint_path = f'{iter_id}.h5'
cb_checkpt = ModelCheckpoint(checkpoint_path, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=True, mode='min')
 ocp = one_cycle_scheduler.OneCycleScheduler(verbose=0, **scheduler_params)
 model.fit(X_train, y_train,
validation_data=(X_valid, y_valid),
callbacks=[cb_checkpt, ocp],
**fit_params)
 model.load_weights(checkpoint_path)
 if pseudo > 0:
if verbose: print(f'{logloss_keras(y_valid, model.predict(X_valid)) :.5f}', end=' ')
 pseudo_preds = model.predict(X_test)[~test_control]
fit_params_p = fit_params.copy() ; fit_params_p['epochs'] = pseudo
model.fit(X_test[~test_control], pseudo_preds,
validation_data=(X_valid, y_valid),
callbacks=[cb_checkpt], **fit_params_p)
 model.load_weights(checkpoint_path)
 fold_preds = model.predict(X_valid)
 preds[valid_index] += fold_preds
oof += model.predict(X_test)
 val_loss = logloss_keras(y_valid, fold_preds)
 history.append(val_loss)
 if verbose: print(f'{val_loss:.5f}')
 tries = len(seeds)* len(splits)
 folds = len(seeds)* sum([n for _, n in splits])
 preds /= tries
oof /= folds
return preds, oof, np.array(history)
 pseudo_epochs = 5
splits = [(MultilabelStratifiedKFold, 6)]
seeds = range(5)
 fold_mask = None
preds, oof, history = crossval(X, y, test, model_params, scheduler_params, fit_params=fit_params, splits=splits, seeds=seeds, fold_mask=fold_mask, pseudo=pseudo_epochs)
 print(history.mean() , history.std() )<set_options>
loss_fn = nn.BCEWithLogitsLoss()
def set_seed(seed_value=42):

 random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 def train(model, train_dataloader, val_dataloader=None, epochs=4, evaluation=False):

 print("Start training...
 ")
 for epoch_i in range(epochs):
print(f"{'Epoch':^7} | {'Batch':^7} | {'Train Loss':^12} | {'Val Loss':^10} | {'Val Acc':^9} | {'Elapsed':^9}")
 print("-"*70)
 t0_epoch, t0_batch = time.time() , time.time()
total_loss, batch_loss, batch_counts = 0, 0, 0
model.train()
for step, batch in enumerate(train_dataloader):
batch_counts +=1
b_input_ids, b_attn_mask, b_labels = tuple(t.to(device)for t in batch)
 optimizer.zero_grad()
logits = model(b_input_ids, b_attn_mask)
 loss = loss_fn(logits, b_labels.float())
 batch_loss += loss.item()
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
if(step % 50000 == 0 and step != 0)or(step == len(train_dataloader)- 1):
time_elapsed = time.time() - t0_batch
print(f"{epoch_i + 1:^7} | {step:^7} | {batch_loss / batch_counts:^12.6f} | {'-':^10} | {'-':^9} | {time_elapsed:^9.2f}")
 batch_loss, batch_counts = 0, 0
t0_batch = time.time()
avg_train_loss = total_loss / len(train_dataloader)
 print("-"*70)
 if evaluation == True:
val_loss, val_accuracy = evaluate(model, val_dataloader)
 time_elapsed = time.time() - t0_epoch
print(f"{epoch_i + 1:^7} | {'-':^7} | {avg_train_loss:^12.6f} | {val_loss:^10.6f} | {val_accuracy:^9.2f} | {time_elapsed:^9.2f}")
 print("-"*70)
 print("
")
 print("Training complete!")
 def evaluate(model, val_dataloader):

 model.eval()
val_accuracy = []
val_loss = []
for batch in val_dataloader:
b_input_ids, b_attn_mask, b_labels = tuple(t.to(device)for t in batch)
 with torch.no_grad() :
logits = model(b_input_ids, b_attn_mask)
 loss = loss_fn(logits, b_labels.float())
 val_loss.append(loss.item())
 accuracy = accuracy_thresh(logits.view(-1,6),b_labels.view(-1,6))
val_accuracy.append(accuracy)
 val_loss = np.mean(val_loss)
 val_accuracy = np.mean(val_accuracy)
 return val_loss, val_accuracy
def accuracy_thresh(y_pred, y_true, thresh:float=0.5, sigmoid:bool=True):
"Compute accuracy when `y_pred` and `y_true` are the same size."
if sigmoid:
y_pred = y_pred.sigmoid()
return(( y_pred>thresh)==y_true.byte() ).float().mean().item()
warnings.filterwarnings('ignore' )<import_modules>
set_seed(42)
 bert_classifier, optimizer, scheduler = initialize_model(epochs=1)
 train(bert_classifier, train_dataloader, val_dataloader, epochs=1, evaluation=True )
from sklearn.preprocessing import QuantileTransformer<import_modules>
def bert_predict(model, test_dataloader):

 model.eval()
all_logits = []
for batch in test_dataloader:
b_input_ids, b_attn_mask = tuple(t.to(device)for t in batch)[:2]
with torch.no_grad() :
logits = model(b_input_ids, b_attn_mask)
 all_logits.append(logits)
 all_logits = torch.cat(all_logits, dim=0)
 probs = all_logits.sigmoid().cpu().numpy()
return probs
import os
import gc
import pickle
import datetime
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
from tqdm.notebook import tqdm
from time import time<import_modules>
probs = bert_predict(bert_classifier,val_dataloader)
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
import tensorflow_addons as tfa
from sklearn.model_selection import KFold
from tensorflow.keras import layers,regularizers,Sequential,backend,callbacks,optimizers,metrics,losses<load_from_csv>
full_train_data = torch.utils.data.ConcatDataset([train_data, val_data])
 full_train_sampler = RandomSampler(full_train_data)
 full_train_dataloader = DataLoader(full_train_data, sampler=full_train_sampler, batch_size=32)
 set_seed(42)
 bert_classifier, optimizer, scheduler = initialize_model(epochs=4)
 train(bert_classifier, full_train_dataloader, epochs=4 )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
print('Tokenizing data...')
 test_inputs, test_masks = preprocessing_for_bert(test.comment_text)
 test_dataset = TensorDataset(test_inputs, test_masks)
 test_dataloader = DataLoader(test_dataset, shuffle=False, batch_size=32 )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<normalization>
probs = bert_predict(bert_classifier, test_dataloader)
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
submission = pd.DataFrame(probs,columns=['toxic','severe_toxic','obscene','threat','insult','identity_hate'])
 test[['toxic','severe_toxic','obscene','threat','insult','identity_hate']]=submission
final_sub = test[['id','toxic','severe_toxic','obscene','threat','insult','identity_hate']]
final_sub.head()
<concatenate><EOS> 
final_sub.to_csv('submissions.csv',index=False)
 final_sub.head()