Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<categorify>
from PIL import Image
import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader, Subset, TensorDataset
from sklearn.model_selection import train_test_split
from tqdm import notebook
import torchvision
from torchvision import transforms
from torchvision import models
from collections import Counter
from pathlib import Path
from sklearn.metrics import confusion_matrix, recall_score
import seaborn as sn
import pyarrow.parquet as pq
from skimage.filters import threshold_otsu
from skimage.transform import AffineTransform, SimilarityTransform, warp, resize
data=pd.get_dummies(data ).reset_index(drop=True)
 data.head(3 )<import_modules>
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
import lightgbm as lgb
from lightgbm import LGBMRegressor
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import make_pipeline 
<choose_model_class>
df_test = pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv' )
model1 = RandomForestRegressor(n_estimators=1000,max_depth=12,min_samples_split=5,min_samples_leaf=5,oob_score=True,random_state=50)
 model2 = XGBRegressor(n_estimators=5000,learning_rate=0.01,early_stopping_round=10,max_depth=4,min_child_weight=0,gamma=0.6,verbose=False,random_state=50)
 model3= LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
verbose=-1,
random_state=50 )<compute_train_metric>
def thresh(t):
t = t.clone().detach()
t.mul_(-1)
 nn.functional.threshold_(t, -200, -255)
 t.mul_(-1)
 return t
def make_tensordataset_from_dfs(parquet_locs, label_loc=None):
ids = []
X = []
for parquet_loc in parquet_locs:
df = pd.read_parquet(parquet_loc)
 ids.extend(df.image_id.tolist())
 x = df.iloc[:, 1:].to_numpy(dtype=np.uint8)
 del df
X.append(x)
 X = np.vstack(X)
 X = X.reshape(-1, 1, 137, 236)
 X = torch.from_numpy(X)
 ids = dict(( s,i)for(i,s)in enumerate(ids))
if label_loc is None:
return TensorDataset(X)
 else:
graphemes = torch.zeros(X.shape[0], dtype=torch.long)
 vowel_diacs = torch.zeros(X.shape[0], dtype=torch.long)
 consonant_diacs = torch.zeros(X.shape[0], dtype=torch.long)
 lbl_df = pd.read_csv(label_loc)
 for row in lbl_df.itertuples() :
if row.image_id not in ids:
continue
idx = ids[row.image_id]
graphemes[idx] = row.grapheme_root
vowel_diacs[idx] = row.vowel_diacritic
consonant_diacs[idx] = row.consonant_diacritic
return TensorDataset(X, graphemes, vowel_diacs, consonant_diacs )
def cv(model,x=train):
er = -1*cross_val_score(model,x,label,cv=10,scoring='neg_mean_absolute_error')
 rmse=np.sqrt(er)
 return rmse<compute_train_metric>
class DatasetWithImageTransforms(Dataset):
def __init__(self, ds, transforms):
super(DatasetWithImageTransforms, self ).__init__()
self.ds = ds
self.tr = transforms
self.nt = len(self.ds[0])
 self.ln = len(self.ds)
 def __getitem__(self, index):
img, *rest = self.ds[index]
img = self.tr(img)
 return(img,)+ tuple(rest)
 def __len__(self):
return self.ln
score1=cv(model1)
 Rf=[score1.mean() ,score1.std() ]
Rf<compute_train_metric>
def thresh(img):
thresh_val = int(threshold_otsu(img))
img =(img > thresh_val)
 return img
def bounding_box(img):
img = thresh(img ).astype(np.uint8)
 col_min_val = np.min(img, axis=0)
 row_min_val = np.min(img, axis=1)
 col = np.argwhere(1 - col_min_val ).flatten()
row = np.argwhere(1 - row_min_val ).flatten()
return row.min() , row.max() , col.min() , col.max()
def scale_to_bb(img):
height = img.shape[0]
width = img.shape[1]
t, b, l, r = bounding_box(img)
 box_width = r - l
box_height = b - t
t, l = max(0, t - 10), max(0, l - 10)
 b, r = min(height, b + 10), min(width, r + 10)
 img = resize(img[t:b, l:r], output_shape=(256, 256), preserve_range=True, order=3, cval=1.0)
 return img
def random_scale(img):
height = img.shape[0]
width = img.shape[1]
t, b, l, r = bounding_box(img)
 box_width = r - l
box_height = b - t
max_width_scale =(box_width + min(l, width - r)) / box_width
max_height_scale =(box_height + min(t, height - b)) / box_height
max_scale = min(max_width_scale, max_height_scale)
 min_scale = min(1.0, 0.25 * max(( height / box_height),(width / box_width)))
 scale = random.uniform(min_scale, max_scale)
 tfm = SimilarityTransform(
 scale=(scale, scale), 
)
 img = warp(img, tfm.inverse, cval=1.0, order=3)
 return img
def random_translate(img):
height = img.shape[0]
width = img.shape[1]
t, b, l, r = bounding_box(img)
 box_width = r - l
box_height = b - t
translate_height = random.uniform(-t, height - b)
 translate_width = random.uniform(-l, width - r)
 tfm = SimilarityTransform(
 translation=(translate_width, translate_height), 
)
 img = warp(img, tfm.inverse, cval=1.0, order=3)
 return img
def random_rotate_and_shear(img):
max_theta = math.pi / 16
theta = random.uniform(-max_theta, max_theta)
 max_shear_theta = math.pi / 8
shear_theta = random.uniform(-max_shear_theta, max_shear_theta)
 tfm = AffineTransform(rotation=theta, shear=shear_theta)
 img = warp(img, tfm.inverse, cval=1.0, order=3)
 return img
def invert_color(t):
t.mul_(-1)
 t.add_(255)
 return t
def affine_transforms(img):
img = img.reshape(137, 236 ).numpy()
img = thresh(img ).astype(np.float32)
 img = random_translate(random_scale(img))
img = img.reshape(1, 137, 236)
 return torch.from_numpy(img)
 def tfms(img):
img = img.reshape(137, 236 ).numpy()
img = thresh(img ).astype(np.float32)
 img = scale_to_bb(img)
 img = img.reshape(1, 256, 256)
 return torch.from_numpy(img)
 def va_tfms(img):
img = img.reshape(137, 236 ).numpy()
img = thresh(img ).astype(np.float32)
 img = img.reshape(1, 137, 236)
 return torch.from_numpy(img)
 affine_transforms = transforms.Lambda(affine_transforms)
 mult = transforms.Lambda(lambda img: img * 255)
 to_float = transforms.Lambda(lambda img: img.float())
 invert_color = transforms.Lambda(invert_color)
 tfms = transforms.Compose([
tfms,
to_float,
] )
score2=cv(model2)
 Xg_mean=score2.mean()
Xg_std=score2.std()
Xg=[Xg_mean,Xg_std]
Xg<compute_train_metric>
n_graphemes, n_vowel_diacs, n_consonant_diacs = 168, 11, 7
score3=cv(model3)
 Lg_mean=score3.mean()
Lg_std=score3.std()
Lg=[Lg_mean,Lg_std]
Lg<train_model>
class MultiTaskNN(nn.Module):
def __init__(self, base, task_predictors):
super(MultiTaskNN, self ).__init__()
self.base = base
self.task_predictors = nn.ModuleList(task_predictors)
 def freeze(self):
for p in self.base.parameters() :
p.requires_grad = False
def unfreeze(self):
for p in self.base.parameters() :
p.requires_grad = True
def forward(self, x):
features = self.base(x)
 preds = [predictor(features)for predictor in self.task_predictors]
return preds
model2.fit(train,label )<predict_on_test>
def load_model(name, num_tasks):
base = torch.jit.load(name + '-base.trace')
 task_predictors = [torch.jit.load(name + '-task-predictor-{}.trace'.format(i)) for i in range(num_tasks)]
return MultiTaskNN(base, task_predictors )
pred= model2.predict(test )<load_from_csv>
model = load_model('/kaggle/input/bhgd-post-exp/r18-post-exp-1', 3 )
sample=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv' )<define_variables>
model = model.to(device)
 model.eval() ;
Id= np.array(range(1461,2920))<create_dataframe>
results = []
out1 =pd.DataFrame({"Id":Id,"SalePrice": pred})
 out1<save_to_csv>
with torch.no_grad() :
for i in range(4):
te_ds = make_tensordataset_from_dfs(
 ['/kaggle/input/bengaliai-cv19' + '/test_image_data_{}.parquet'.format(i)] 
)
 te_ds = DatasetWithImageTransforms(te_ds, tfms)
 num_tta = 1
te_dl = DataLoader(te_ds, batch_size=256, num_workers=2, pin_memory=True)
 tta_results = [[] for i in range(num_tta)]
for i in range(num_tta):
for imgs, in te_dl:
imgs = imgs.to(device)
 g_pred, v_pred, c_pred, *rest = model(imgs)
 tta_results[i].append([g_pred, v_pred, c_pred])
 for i in range(len(te_dl)) :
g_preds = [results[i][0] for results in tta_results]
v_preds = [results[i][1] for results in tta_results]
c_preds = [results[i][2] for results in tta_results]
g_pred = torch.stack(g_preds ).mean(dim=0)
 v_pred = torch.stack(v_preds ).mean(dim=0)
 c_pred = torch.stack(c_preds ).mean(dim=0)
 g_pred = g_pred.argmax(1 ).tolist()
v_pred = v_pred.argmax(1 ).tolist()
c_pred = c_pred.argmax(1 ).tolist()
for g, v, c in zip(g_pred, v_pred, c_pred):
results.extend([c, 0, v])
 del imgs
del te_ds
del te_dl
del tta_results
out1.to_csv('my_submission.csv', index=False )<load_from_csv>
del model
train_df = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_df = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_PassengerId = test_df["PassengerId"]<count_values>
submission_df = pd.read_csv('/kaggle/input/bengaliai-cv19' + '/sample_submission.csv' )
category2 = ["Cabin", "Name", "Ticket"]
for c in category2:
print("{}
".format(train_df[c].value_counts()))<sort_values>
submission_df['target'] = results
<sort_values><EOS> 
submission_df.to_csv('./submission.csv', index=False )