Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<sort_values>
!pip install /kaggle/input/timm-model/timm-0.3.2-py3-none-any.whl
train_df[["SibSp","Survived"]].groupby(["SibSp"], as_index = False ).mean().sort_values(by="Survived",ascending = False )<sort_values>
import pandas as pd
import cv2
import numpy as np
from tqdm.auto import tqdm
from matplotlib import pyplot as plt
import time
import pyarrow.parquet as pq
<define_variables><EOS> 
class MultiHeadSimpleModel(nn.Module):
def __init__(self, input_size, backbone, pretrained=True, dropout=0.2):
super(MultiHeadSimpleModel, self ).__init__()
self.input_size = input_size
self.backbone = backbone
self.pretrained = pretrained
self.dropout=dropout
self.backbone = create_model(self.backbone, self.pretrained, num_classes=0)
 feature_size = self.backbone.state_dict() ['bn2.weight'].shape[0]
self.root_head = nn.Linear(feature_size, 168, bias=True)
 self.consonant_head = nn.Linear(feature_size, 168, bias=True)
 self.vowel_head = nn.Linear(feature_size, 168, bias=True)
 intermid = []
intermid.append(nn.BatchNorm1d(feature_size))
intermid.append(nn.Dropout(self.dropout))
intermid.append(nn.Linear(feature_size, 512))
intermid.append(nn.BatchNorm1d(512))
self.intermid_unique = nn.ModuleList(intermid)
 torch.nn.init.kaiming_normal_(self.intermid_unique[2].weight)
 self.arc_face_unique = Arcface(512, 1295)
 def multi_head(self, input, unique):
input = self.backbone(input)
 root = self.root_head(input)
 consonant = self.consonant_head(input)
 vowel = self.vowel_head(input)
 x = self.intermid_unique[0](input)
 for inter in self.intermid_unique[1:]:
x = inter(x)
 x = F.normalize(x, p=2, dim=1)
 unique_p = self.arc_face_unique(x, unique)
 return root, consonant, vowel, unique_p
def forward(self, input, unique):
multi_head_outputs = self.multi_head(input, unique)
 return multi_head_outputs
def l2_norm(input,axis=1):
norm = torch.norm(input,2,axis,True)
 output = torch.div(input, norm)
 return output
class Arcface(nn.Module):
def __init__(self, embedding_size=512, classnum=51332, s=30., m=0.5):
super(Arcface, self ).__init__()
self.classnum = classnum
self.weight = Parameter(torch.Tensor(embedding_size,classnum))
self.weight.data.uniform_(-1, 1 ).renorm_(2,1,1e-5 ).mul_(1e5)
 self.m = m
self.s = s
self.cos_m = math.cos(m)
 self.sin_m = math.sin(m)
 self.mm = self.sin_m * m
self.threshold = math.cos(math.pi - m)
 def forward(self, embbedings, label):
nB = len(embbedings)
 kernel_norm = l2_norm(self.weight,axis=0)
 cos_theta = torch.mm(embbedings,kernel_norm)
 cos_theta = cos_theta.clamp(-1,1)
 cos_theta_2 = torch.pow(cos_theta, 2)
 sin_theta_2 = 1 - cos_theta_2
sin_theta = torch.sqrt(sin_theta_2)
 cos_theta_m =(cos_theta * self.cos_m - sin_theta * self.sin_m)
 cond_v = cos_theta - self.threshold
cond_mask = cond_v <= 0
keep_val =(cos_theta - self.mm)
 cos_theta_m[cond_mask] = keep_val[cond_mask]
output = cos_theta * 1.0
idx_ = torch.arange(0, nB, dtype=torch.long)
 output[idx_, label] = cos_theta_m[idx_, label]
output *= self.s
return output
class BengaliDataset(Dataset):
def __init__(self, label_csv, unique_csv, train_folder, parquet_path, transforms, cache=True, test=True):
self.label_csv = label_csv
self.unique_csv = unique_csv
self.train_folder = train_folder
self.parquet_path = parquet_path
self.label = pd.read_csv(self.label_csv)
 self.label = self.label[self.label['component']=='grapheme_root']
self.label = self.label.reset_index(drop=True)
 unique_df = pd.read_csv(self.unique_csv)
 self.names = self.label['image_id'].values
self.uniques = unique_df.grapheme.unique()
self.transforms = transforms
self.img = [None] * self.label.shape[0]
self.test = test
if cache:
self.cache_images()
def cache_images(self):
count = 0
root = './test_graphemes/'
self.p = p = pd.read_parquet(self.parquet_path ).drop(['image_id'], axis=1)
 def load_image(self, idx):
img = resize(self.p.iloc[[idx]] ).values.reshape(-1 ).reshape(IMG_SIZE, IMG_SIZE ).astype(np.uint8)
 return self.transforms(img)
 img = self.img[idx]
if img is None:
img = resize(p.iloc[[i]] ).values.reshape(-1 ).reshape(IMG_SIZE, IMG_SIZE ).astype(np.uint8)
 return self.transforms(img)
 else:
return self.transforms(img)
 def __getitem__(self, idx):
if not self.test:
img = self.load_image(idx)
 root = self.label.loc[idx]['grapheme_root']
consonant = self.label.loc[idx]['consonant_diacritic']
vowel = self.label.loc[idx]['vowel_diacritic']
unique = np.where(self.uniques == self.label.grapheme[idx])[0][0]
return transforms.ToTensor()(img), root, consonant, vowel, unique
else:
img = self.load_image(idx)
 root = 0
consonant = 0
vowel = 0
unique = 0
return transforms.ToTensor()(img), root, consonant, vowel, unique
def __len__(self):
return self.p.shape[0]
def postprocess(preds, num_classes, EXP = -1.2):
p0 = np.argmax(preds,axis=1)
 s = pd.Series(p0)
 vc = s.value_counts().sort_index()
df = pd.DataFrame({'a':np.arange(num_classes),'b':np.ones(num_classes)})
 df.b = df.a.map(vc)
 df.fillna(df.b.min() ,inplace=True)
 mat1 = np.diag(df.b.astype('float32')**EXP)
 p0 = np.argmax(preds.dot(mat1), axis=1)
 return p0
class Tester:
def __init__(self,
dataset_path='./drive/MyDrive/datasets/car classification/train_data',
batch_size=1,
model_name='tf_efficientnet_b3_ns',
test_csv='./train_labels.csv',
unique_csv='./train_labels.csv',
output_dir='.. /drive/MyDrive/ckpt/grapheme/submission.csv',
ckpt='.. /drive/MyDrive/ckpt/grapheme/20.pth',
multihead_ckpt='.. /'):
self.dataset_path = dataset_path
self.batch_size = batch_size
self.model_name = model_name
self.test_csv = test_csv
self.unique_csv = unique_csv
self.output_dir = output_dir
self.ckpt = ckpt
self.multihead_ckpt = multihead_ckpt
if model_name == 'tf_efficientnet_b0_ns':
self.input_size =(224, 224)
 elif model_name == 'tf_efficientnet_b2_ns':
self.input_size =(260, 260)
 elif model_name == 'tf_efficientnet_b3_ns':
self.input_size =(300, 300)
 elif model_name == 'tf_efficientnet_b4_ns':
self.input_size =(380, 380)
 elif model_name == 'tf_efficientnet_b6_ns':
self.input_size =(528, 528)
 else:
raise Exception('non-valid model name')
 transform = []
transform += [transforms.ToPILImage() ]
transform += [transforms.Resize(self.input_size)]
self.transform = transforms.Compose(transform)
 self.device = torch.device('cuda')
 self.model_root = create_model(self.model_name, pretrained=False, num_classes=168 ).to(self.device)
 self.model_consonant = create_model(self.model_name, pretrained=False, num_classes=8 ).to(self.device)
 self.model_vowel = create_model(self.model_name, pretrained=False, num_classes=11 ).to(self.device)
 self.model_multihead = MultiHeadSimpleModel(self.input_size, self.model_name, pretrained=False, dropout=0 ).to(self.device)
 ckpt = torch.load(self.ckpt)
 multihead_ckpt = torch.load(self.multihead_ckpt)
 self.model_root.load_state_dict(ckpt['model_root_state_dict'])
 self.model_consonant.load_state_dict(ckpt['model_consonant_state_dict'])
 self.model_vowel.load_state_dict(ckpt['model_vowel_state_dict'])
 self.model_multihead.load_state_dict(multihead_ckpt['model_multihead_state_dict'])
 def test(self):
output_roots = []
output_consonants = []
output_vowels = []
count = 0
for ii in range(4):
self.test_dataset = BengaliDataset(self.test_csv, self.unique_csv, self.dataset_path, f'/kaggle/input/bengaliai-cv19/test_image_data_{ii}.parquet', self.transform, cache=True)
 self.names = self.test_dataset.names
self.test_dataloader = DataLoader(self.test_dataset, batch_size=self.batch_size, num_workers=0, shuffle=False)
 pbar = tqdm.tqdm(self.test_dataloader)
 pbar.set_description('testing process')
 self.model_root.eval()
self.model_consonant.eval()
self.model_vowel.eval()
self.model_multihead.eval()
with torch.no_grad() :
for it, data in enumerate(pbar):
inputs = data[0].to(self.device)
 inputs = inputs.repeat(1, 3, 1, 1)
 roots = data[1].to(self.device ).long()
consonants = data[2].to(self.device ).long()
vowels = data[3].to(self.device ).long()
uniques = data[4].to(self.device ).long()
root_preds = self.model_root(inputs)
 self.model_root.zero_grad()
consonant_preds = self.model_consonant(inputs)
 self.model_consonant.zero_grad()
vowel_preds = self.model_vowel(inputs)
 self.model_vowel.zero_grad()
root, consonant, vowel, unique = self.model_multihead(inputs, uniques)
 root_preds = postprocess(torch.nn.Softmax(1 )(root_preds ).cpu().numpy() , 168, -0.9)
 consonant_preds = postprocess(torch.nn.Softmax(1 )(consonant_preds ).cpu().numpy() , 8, -0.6)
 vowel_preds = postprocess(torch.nn.Softmax(1 )(vowel_preds ).cpu().numpy() , 11, -1.1)
 root = postprocess(torch.nn.Softmax(1 )(root ).cpu().numpy() , 168, -1.2)
 consonant = postprocess(torch.nn.Softmax(1 )(consonant[:,:8] ).cpu().numpy() , 8, -0.5)
 vowel = postprocess(torch.nn.Softmax(1 )(vowel[:,:11] ).cpu().numpy() , 11, -1.2)
 unique_prob = F.softmax(unique, dim=1)
 for index in range(inputs.shape[0]):
if unique_prob[index].max(-1 ).values < 0.5:
output_roots.append(root_preds[index].item())
 output_consonants.append(consonant_preds[index].item())
 output_vowels.append(vowel_preds[index].item())
 else:
output_roots.append(root[index].item())
 output_consonants.append(consonant[index].item())
 output_vowels.append(vowel[index].item())
 del self.test_dataset.p
del self.test_dataset
del self.test_dataloader
row_id, target = [], []
for iid, r, c, v in zip(self.names, output_roots, output_consonants, output_vowels):
row_id.append(iid + '_grapheme_root')
 target.append(int(r))
row_id.append(iid + '_vowel_diacritic')
 target.append(int(v))
row_id.append(iid + '_consonant_diacritic')
 target.append(int(c))
count += 1
sub_fn = self.output_dir
sub = pd.DataFrame({'row_id': row_id, 'target': target})
 sub.to_csv(sub_fn, index=False)
 print(f'Done wrote to {sub_fn}')
 tester = Tester(dataset_path='./test_graphemes',
batch_size=128,
model_name='tf_efficientnet_b2_ns',
test_csv='/kaggle/input/bengaliai-cv19/test.csv',
unique_csv='/kaggle/input/bengaliai-cv19/train.csv',
output_dir='./submission.csv',
ckpt='/kaggle/input/graphemesingle/27.pth',
multihead_ckpt='/kaggle/input/graphemesingle/10.pth' 
)
 tester.test()