Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<load_from_csv>
import os
import warnings
import logging
from typing import Mapping, List, Union, Optional, Tuple
from pprint import pprint
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from transformers import AutoConfig, AutoModel, AutoTokenizer
import catalyst
from catalyst.dl import SupervisedRunner
from catalyst.dl.callbacks import OptimizerCallback, CheckpointCallback, InferCallback
from catalyst.dl.utils import plot_metrics
from catalyst.utils import set_global_seed, prepare_cudnn
best = pd.read_csv(".. /input/accuracy-best-public-lbs/kkiller_first_public_notebook_under050_v5.csv")
 best.head()<load_from_csv>
catalyst.__version__
sales = pd.read_csv(".. /input/m5-forecasting-uncertainty/sales_train_validation.csv")
 sales.head()<merge>
MODEL_NAME = 'distilbert-base-uncased'
LOG_DIR = "./logdir"
NUM_EPOCHS = 3
BATCH_SIZE = 96
MAX_SEQ_LENGTH = 256
LEARN_RATE = 3e-5
ACCUM_STEPS = 4
SEED = 17
sub = best.merge(sales[["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]], on = "id")
 sub["_all_"] = "Total"
sub.shape<define_search_space>
PATH_TO_DATA = '.. /input/jigsaw-toxic-comment-classification-challenge/'
TEXT_FIELD = 'comment_text'
TARGET_FIELDS = ['toxic','severe_toxic','obscene','threat','insult', 'identity_hate']
NUM_CLASSES = len(TARGET_FIELDS)
 PRED_THRES = 0.4
qs = np.array([0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995])
 qs.shape
def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 level_coef_dict = {"id": get_ratios(coef=0.3), "item_id": get_ratios(coef=0.15),
"dept_id": get_ratios(coef=0.08), "cat_id": get_ratios(coef=0.07),
"store_id": get_ratios(coef=0.08), "state_id": get_ratios(coef=0.07), "_all_": get_ratios(coef=0.05), 
("state_id", "item_id"): get_ratios(coef=0.19),("state_id", "dept_id"): get_ratios(coef=0.1), 
("store_id","dept_id"): get_ratios(coef=0.11),("state_id", "cat_id"): get_ratios(coef=0.08), 
("store_id","cat_id"): get_ratios(coef=0.1)
 }<statistical_test>
train_df = pd.read_csv(PATH_TO_DATA + 'train.csv.zip', index_col='id')
 test_df = pd.read_csv(PATH_TO_DATA + 'test.csv.zip', index_col='id' )
def quantile_coefs(q, level):
ratios = level_coef_dict[level]
return ratios.loc[q].values
def get_group_preds(pred, level):
df = pred.groupby(level)[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q, level)[:, None]
if level != "id":
df["id"] = [f"{lev}_X_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
else:
df["id"] = [f"{lev.replace('_validation', '')}_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
df = df[["id"]+list(cols)]
return df
def get_couple_group_preds(pred, level1, level2):
df = pred.groupby([level1, level2])[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q,(level1, level2)) [:, None]
df["id"] = [f"{lev1}_{lev2}_{q:.3f}_validation" for lev1,lev2, q in
zip(df[level1].values,df[level2].values, q)]
df = df[["id"]+list(cols)]
return df
levels = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id", "_all_"]
couples = [("state_id", "item_id"),("state_id", "dept_id"),("store_id","dept_id"), 
("state_id", "cat_id"),("store_id","cat_id")]
cols = [f"F{i}" for i in range(1, 29)]
df = []
for level in levels :
df.append(get_group_preds(sub, level))
for level1,level2 in couples:
df.append(get_couple_group_preds(sub, level1, level2))
df = pd.concat(df, axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df = pd.concat([df,df] , axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df.loc[df.index >= len(df.index)//2, "id"] = df.loc[df.index >= len(df.index)//2, "id"].str.replace(
 "_validation$", "_evaluation")
 df.shape
df.head()
df.to_csv("submission01.csv", index = False )<import_modules>
X_train, X_valid, y_train, y_valid = train_test_split(train_df[TEXT_FIELD],
train_df[TARGET_FIELDS],
test_size=0.1,
random_state=17)
 X_test = test_df[TEXT_FIELD]
import pandas as pd, numpy as np
from matplotlib import pyplot as plt
import scipy.stats as stats<set_options>
class TextClassificationDataset(Dataset):

 def __init__(self,
texts: List[str],
labels: np.ndarray = None,
max_seq_length: int = 512,
model_name: str = 'distilbert-base-uncased'):

 self.texts = texts
self.labels = labels
self.max_seq_length = max_seq_length
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
 logging.getLogger(
 "transformers.tokenization_utils" ).setLevel(logging.FATAL)
 def __len__(self):

 return len(self.texts)
 def __getitem__(self, index)-> Mapping[str, torch.Tensor]:

 x = self.texts[index]
output_dict = self.tokenizer.encode_plus(
 x,
add_special_tokens=True,
pad_to_max_length=True,
max_length=self.max_seq_length,
return_tensors="pt",
return_attention_mask=True 
)
 output_dict['features'] = output_dict['input_ids'].squeeze(0)
 del output_dict['input_ids']
if self.labels is not None:
output_dict["targets"] = torch.from_numpy(self.labels[index] ).float()
return output_dict
pd.options.display.max_columns = 50<load_from_csv>
train_dataset = TextClassificationDataset(
 texts=X_train.values.tolist() ,
labels=y_train.values,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME 
)
 valid_dataset = TextClassificationDataset(
 texts=X_valid.values.tolist() ,
labels=y_valid.values,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME 
)
 test_dataset = TextClassificationDataset(
 texts=X_test.values.tolist() ,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME
)
best = pd.read_csv(".. /input/accuracy-best-public-lbs/kkiller_first_public_notebook_under050_v5.csv")
 best.head()<load_from_csv>
train_val_loaders = {
"train": DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=True),
"valid": DataLoader(dataset=valid_dataset,
batch_size=BATCH_SIZE,
shuffle=False)
 }
sales = pd.read_csv(".. /input/m5-forecasting-uncertainty/sales_train_validation.csv")
 sales.head()<merge>
class BertForSequenceClassification(nn.Module):

 def __init__(self, pretrained_model_name: str, num_classes: int = None, dropout: float = 0.3):

 super().__init__()
config = AutoConfig.from_pretrained(
 pretrained_model_name, num_labels=num_classes)
 self.model = AutoModel.from_pretrained(pretrained_model_name,
config=config)
 self.classifier = nn.Linear(config.hidden_size, num_classes)
 self.dropout = nn.Dropout(dropout)
 def forward(self, features, attention_mask=None, head_mask=None):

 assert attention_mask is not None, "attention mask is none"
bert_output = self.model(input_ids=features,
attention_mask=attention_mask,
head_mask=head_mask)
 seq_output = bert_output[0]
pooled_output = seq_output.mean(axis=1)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 return logits
sub = best.merge(sales[["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]], on = "id")
 sub["_all_"] = "Total"
sub.shape<define_search_space>
model = BertForSequenceClassification(pretrained_model_name=MODEL_NAME,
num_classes=NUM_CLASSES )
qs = np.array([0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995])
 qs.shape
def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 level_coef_dict = {"id": get_ratios(coef=0.3), "item_id": get_ratios(coef=0.15),
"dept_id": get_ratios(coef=0.08), "cat_id": get_ratios(coef=0.07),
"store_id": get_ratios(coef=0.08), "state_id": get_ratios(coef=0.07), "_all_": get_ratios(coef=0.05), 
("state_id", "item_id"): get_ratios(coef=0.19),("state_id", "dept_id"): get_ratios(coef=0.1), 
("store_id","dept_id"): get_ratios(coef=0.11),("state_id", "cat_id"): get_ratios(coef=0.08), 
("store_id","cat_id"): get_ratios(coef=0.1)
 } 
<statistical_test>
criterion = torch.nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=LEARN_RATE)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer )
def quantile_coefs(q, level):
ratios = level_coef_dict[level]
return ratios.loc[q].values
def get_group_preds(pred, level):
df = pred.groupby(level)[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q, level)[:, None]
if level != "id":
df["id"] = [f"{lev}_X_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
else:
df["id"] = [f"{lev.replace('_validation', '')}_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
df = df[["id"]+list(cols)]
return df
def get_couple_group_preds(pred, level1, level2):
df = pred.groupby([level1, level2])[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q,(level1, level2)) [:, None]
df["id"] = [f"{lev1}_{lev2}_{q:.3f}_validation" for lev1,lev2, q in
zip(df[level1].values,df[level2].values, q)]
df = df[["id"]+list(cols)]
return df
levels = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id", "_all_"]
couples = [("state_id", "item_id"),("state_id", "dept_id"),("store_id","dept_id"), 
("state_id", "cat_id"),("store_id","cat_id")]
cols = [f"F{i}" for i in range(1, 29)]
df = []
for level in levels :
df.append(get_group_preds(sub, level))
for level1,level2 in couples:
df.append(get_couple_group_preds(sub, level1, level2))
df = pd.concat(df, axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df = pd.concat([df,df] , axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df.loc[df.index >= len(df.index)//2, "id"] = df.loc[df.index >= len(df.index)//2, "id"].str.replace(
 "_validation$", "_evaluation")
 df.shape
df.head()
df.to_csv("submission01.csv", index = False )<define_variables>
def preprocess_multi_label_metrics(
 outputs: torch.Tensor,
targets: torch.Tensor,
weights: Optional[torch.Tensor] = None, 
)-> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:

 if not torch.is_tensor(outputs):
outputs = torch.from_numpy(outputs)
 if not torch.is_tensor(targets):
targets = torch.from_numpy(targets)
 if weights is not None:
if not torch.is_tensor(weights):
weights = torch.from_numpy(weights)
 weights = weights.squeeze()
if outputs.dim() == 1:
outputs = outputs.view(-1, 1)
 else:
assert outputs.dim() == 2,(
 "wrong `outputs` size "
"(should be 1D or 2D with one column per class)" 
)
 if targets.dim() == 1:
targets = targets.view(-1, 1)
 else:
assert targets.dim() == 2,(
 "wrong `targets` size "
"(should be 1D or 2D with one column per class)" 
)
 if weights is not None:
assert weights.dim() == 1, "Weights dimension should be 1"
assert weights.numel() == targets.size(
 0 
), "Weights dimension 1 should be the same as that of target"
assert torch.min(weights)>= 0, "Weight should be non-negative only"
assert torch.equal(
 targets ** 2, targets 
), "targets should be binary(0 or 1)"
return outputs, targets, weights
def multi_label_accuracy(
 outputs: torch.Tensor,
targets: torch.Tensor,
threshold: Union[float, torch.Tensor],
activation: Optional[str] = None, 
)-> torch.Tensor:

 outputs, targets, _ = preprocess_multi_label_metrics(
 outputs=outputs, targets=targets 
)
 activation_fn = get_activation_fn(activation)
 outputs = activation_fn(outputs)
 outputs =(outputs > threshold ).long()
output =(targets.long() == outputs.long() ).sum().float() / np.prod(
 targets.shape 
)
 return output
class MultiLabelAccuracyCallback(MetricCallback):

 def __init__(
 self,
input_key: str = "targets",
output_key: str = "logits",
prefix: str = "multi_label_accuracy",
threshold: float = None,
activation: str = "Sigmoid", 
):

 super().__init__(
 prefix=prefix,
metric_fn=multi_label_accuracy,
input_key=input_key,
output_key=output_key,
threshold=threshold,
activation=activation,
)
pd.set_option('display.max_columns', 500)
 print("XGBoost version:", xgb.__version__)
 env = janestreet.make_env()
iter_test = env.iter_test()<import_modules>
os.environ['CUDA_VISIBLE_DEVICES'] = "0"
set_global_seed(SEED)
 prepare_cudnn(deterministic=True )
%%time 
<load_from_csv>
%%time
runner = SupervisedRunner(
 input_key=(
 "features",
"attention_mask" 
)
)
 runner.train(
 model=model,
criterion=criterion,
optimizer=optimizer,
scheduler=scheduler,
loaders=train_val_loaders,
callbacks=[
MultiLabelAccuracyCallback(threshold=PRED_THRES),
OptimizerCallback(accumulation_steps=ACCUM_STEPS)
 ],
logdir=LOG_DIR,
num_epochs=NUM_EPOCHS,
verbose=True
)
%%time
train_cudf = cu.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 data = train_cudf.to_pandas()
del train_cudf<data_type_conversions>
!nvidia-smi
features = [c for c in data.columns if 'feature' in c]
for i in features:
x = data[i].mean()
data[i] = data[i].fillna(x )<prepare_x_and_y>
torch.cuda.empty_cache()
<prepare_x_and_y>
!nvidia-smi
print(data.shape)
 data = data[data['weight']!=0]
data['action'] =(data['resp']>0)*1
print(data.shape)
 train = data.sample(frac = 0.75, random_state = 73)
 print(train.shape)
 valid = data.drop(train.index)
 print(valid.shape)
 train.reset_index(drop=True, inplace=True)
 valid.reset_index(drop=True, inplace=True)
 X_train = train.loc[:,features]
y_train = train.loc[:, 'action']
X_valid = valid.loc[:, features]
y_valid = valid.loc[:, 'action']<train_model>
test_loaders = {
"test": DataLoader(dataset=test_dataset,
batch_size=BATCH_SIZE,
shuffle=False)
 }
%%time
selector = CatBoostClassifier(thread_count = -1, task_type = "GPU", devices = '-1', random_seed = 73 ).fit(Pool(X_train, y_train), verbose = 100)
 list_of_tuples = list(zip(X_train.columns.values, selector.get_feature_importance()))
df = pd.DataFrame(list_of_tuples ).sort_values(by = [1] ).reset_index(drop = True ).rename(columns = {0: 'feat_labels', 1: 'feature_importances'})
 df.head()<train_model>
%%time
runner.infer(
 model=model,
loaders=test_loaders,
callbacks=[
CheckpointCallback(
 resume=f"{LOG_DIR}/checkpoints/best.pth" 
),
InferCallback() ,
],
verbose=True
)
model = xgb.XGBClassifier(
 n_estimators=500,
max_depth=11,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.7,
missing=None,
random_state=73,
tree_method='gpu_hist' 
)
 %time model.fit(X_train, y_train )<predict_on_test>
predicted_probs = runner.callbacks[0].predictions['logits']
for(test_df, sample_prediction_df)in tqdm(iter_test):
X_test = test_df.loc[:, features]
for i in features:
x = X_test[i].mean()
X_test[i] = X_test[i].fillna(x)
 y_preds = model.predict(X_test)
 sample_prediction_df.action = y_preds
env.predict(sample_prediction_df )<import_modules>
sample_sub_df = pd.read_csv(PATH_TO_DATA + 'sample_submission.csv.zip',
index_col='id' )
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices
import kerastuner as kt<import_modules>
sample_sub_df[TARGET_FIELDS] = predicted_probs
<split><EOS> 
sample_sub_df.to_csv('submissions.csv' )