Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<merge>
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
All_data = All_data.merge(base, on='Patient', how='left')
 All_data['base_week'] = All_data['Weeks'] - All_data['min_week']<data_type_conversions>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in All_data[col].unique() :
FE.append(mod)
 All_data[mod] =(All_data[col] == mod ).astype(int)
<categorify>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
def csv_normlize(data,attr1,attr):
data[attr1] =(data[attr] - data[attr].min())/(data[attr].max() - data[attr].min())
 csv_normlize(All_data,'age','Age')
 csv_normlize(All_data,'BASE','min_FVC')
 csv_normlize(All_data,'week','base_week')
 csv_normlize(All_data,'percent','Percent')
 FE += ['age','percent','week','BASE']<define_variables>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
root_dir = '/kaggle/input/osic-pulmonary-fibrosis-progression/train'
test_dir = '/kaggle/input/osic-pulmonary-fibrosis-progression/test'
desk_dir = 'save_output'
model_file = '/kaggle/working/diophantus.pt'
resize_dims =(40, 256, 256)
 clip_bounds =(-1000, 200)
 watershed_iterations = 1
pre_calculated_mean = 0.02865046213070556
latent_features = 3
batch_size = 16
learning_rate = 1e-4
num_epochs = 3
val_size = 0.2
tensorboard_dir = '/kaggle/working/runs' 
<categorify>
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
class CTScansDataset(Dataset):
def __init__(self, root_dir, transform=None):
self.root_dir = Path(root_dir)
 self.patients = [p for p in self.root_dir.glob('*')if p.is_dir() ]
self.transform = transform
def __len__(self):
return len(self.patients)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
image, metadata = self.load_scan(self.patients[idx])
 sample = {'image': image, 'metadata': metadata}
if self.transform:
sample = self.transform(sample)
 return sample
def save(self, path):
t0 = time()
Path(path ).mkdir(exist_ok=True, parents=True)
 print('Saving pre-processed dataset to disk')
 sleep(1)
 cum = 0
bar = trange(len(self))
for i in bar:
try:
sample = self[i]
image, data = sample['image'], sample['metadata']
cum += torch.mean(image ).item()
bar.set_description(f'Saving CT scan {data.PatientID}')
 fname = Path(path)/ f'{data.PatientID}.pt'
torch.save(image, fname)
 except:
print(i)
 pass
sleep(1)
 bar.close()
print(f'Done! Time {timedelta(seconds=time() - t0)}
'
f'Tot value: {cum }')
 def get_patient(self, patient_id):
patient_ids = [str(p.stem)for p in self.patients]
return self.__getitem__(patient_ids.index(patient_id))
@staticmethod
def load_scan(path):
slices = [pydicom.read_file(p)for p in path.glob('*.dcm')]
try:
slices.sort(key=lambda x: float(x.ImagePositionPatient[2]))
except AttributeError:
warnings.warn(f'Patient {slices[0].PatientID} CT scan does not '
f'have "ImagePositionPatient".Assuming filenames '
f'in the right scan order.')
 image = np.stack([s.pixel_array.astype(float)for s in slices])
 return image, slices[0]
class CropBoundingBox:
@staticmethod
def bounding_box(img3d: np.array):
mid_img = img3d[int(img3d.shape[0] / 2)]
same_first_row =(mid_img[0, :] == mid_img[0, 0] ).all()
same_first_col =(mid_img[:, 0] == mid_img[0, 0] ).all()
if same_first_col and same_first_row:
return True
else:
return False
def __call__(self, sample):
image, data = sample['image'], sample['metadata']
if not self.bounding_box(image):
return sample
mid_img = image[int(image.shape[0] / 2)]
r_min, r_max = None, None
c_min, c_max = None, None
for row in range(mid_img.shape[0]):
if not(mid_img[row, :] == mid_img[0, 0] ).all() and r_min is None:
r_min = row
if(mid_img[row, :] == mid_img[0, 0] ).all() and r_max is None \
and r_min is not None:
r_max = row
break
for col in range(mid_img.shape[1]):
if not(mid_img[:, col] == mid_img[0, 0] ).all() and c_min is None:
c_min = col
if(mid_img[:, col] == mid_img[0, 0] ).all() and c_max is None \
and c_min is not None:
c_max = col
break
image = image[:, r_min:r_max, c_min:c_max]
return {'image': image, 'metadata': data}
class ConvertToHU:
def __call__(self, sample):
image, data = sample['image'], sample['metadata']
img_type = data.ImageType
is_hu = img_type[0] == 'ORIGINAL' and not(img_type[2] == 'LOCALIZER')
 intercept = data.RescaleIntercept
slope = data.RescaleSlope
image =(image * slope + intercept ).astype(np.int16)
 return {'image': image, 'metadata': data}
class Resize:
def __init__(self, output_size):
assert isinstance(output_size, tuple)
 self.output_size = output_size
def __call__(self, sample):
image, data = sample['image'], sample['metadata']
resize_factor = np.array(self.output_size)/ np.array(image.shape)
 image = zoom(image, resize_factor, mode='nearest')
 return {'image': image, 'metadata': data}

 class Clip:
def __init__(self, bounds=(-1000, 500)) :
self.min = min(bounds)
 self.max = max(bounds)
 def __call__(self, sample):
image, data = sample['image'], sample['metadata']
image[image < self.min] = self.min
image[image > self.max] = self.max
return {'image': image, 'metadata': data}
class MaskWatershed:
def __init__(self, min_hu, iterations, show_tqdm):
self.min_hu = min_hu
self.iterations = iterations
self.show_tqdm = show_tqdm
def __call__(self, sample):
image, data = sample['image'], sample['metadata']
stack = []
if self.show_tqdm:
bar = trange(image.shape[0])
 bar.set_description(f'Masking CT scan {data.PatientID}')
 else:
bar = range(image.shape[0])
 for slice_idx in bar:
sliced = image[slice_idx]
stack.append(self.seperate_lungs(sliced, self.min_hu,
self.iterations))
return {
'image': np.stack(stack),
'metadata': sample['metadata']
}
@staticmethod
def seperate_lungs(image, min_hu, iterations):
h, w = image.shape[0], image.shape[1]
marker_internal, marker_external, marker_watershed = MaskWatershed.generate_markers(image)
 sobel_filtered_dx = ndimage.sobel(image, 1)
 sobel_filtered_dy = ndimage.sobel(image, 0)
 sobel_gradient = np.hypot(sobel_filtered_dx, sobel_filtered_dy)
 sobel_gradient *= 255.0 / np.max(sobel_gradient)
 watershed = morphology.watershed(sobel_gradient, marker_watershed)
 outline = ndimage.morphological_gradient(watershed, size=(3,3))
outline = outline.astype(bool)
 blackhat_struct = [[0, 0, 1, 1, 1, 0, 0],
[0, 1, 1, 1, 1, 1, 0],
[1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1],
[0, 1, 1, 1, 1, 1, 0],
[0, 0, 1, 1, 1, 0, 0]]
blackhat_struct = ndimage.iterate_structure(blackhat_struct, iterations)
 outline += ndimage.black_tophat(outline, structure=blackhat_struct)
 lungfilter = np.bitwise_or(marker_internal, outline)
 lungfilter = ndimage.morphology.binary_closing(lungfilter, structure=np.ones(( 5,5)) , iterations=3)
 segmented = np.where(lungfilter == 1, image, min_hu * np.ones(( h, w)))
 return segmented
@staticmethod
def generate_markers(image, threshold=-400):
h, w = image.shape[0], image.shape[1]
marker_internal = image < threshold
marker_internal = segmentation.clear_border(marker_internal)
 marker_internal_labels = measure.label(marker_internal)
 areas = [r.area for r in measure.regionprops(marker_internal_labels)]
areas.sort()
if len(areas)> 2:
for region in measure.regionprops(marker_internal_labels):
if region.area < areas[-2]:
for coordinates in region.coords:
marker_internal_labels[coordinates[0], coordinates[1]] = 0
marker_internal = marker_internal_labels > 0
external_a = ndimage.binary_dilation(marker_internal, iterations=10)
 external_b = ndimage.binary_dilation(marker_internal, iterations=55)
 marker_external = external_b ^ external_a
marker_watershed = np.zeros(( h, w), dtype=np.int)
 marker_watershed += marker_internal * 255
marker_watershed += marker_external * 128
return marker_internal, marker_external, marker_watershed
class Normalize:
def __init__(self, bounds=(-1000, 500)) :
self.min = min(bounds)
 self.max = max(bounds)
 def __call__(self, sample):
image, data = sample['image'], sample['metadata']
image = image.astype(np.float)
 image =(image - self.min)/(self.max - self.min)
 return {'image': image, 'metadata': data}
class ToTensor:
def __init__(self, add_channel=True):
self.add_channel = add_channel
def __call__(self, sample):
image, data = sample['image'], sample['metadata']
if self.add_channel:
image = np.expand_dims(image, axis=0)
 return {'image': torch.from_numpy(image), 'metadata': data}
class ZeroCenter:
def __init__(self, pre_calculated_mean):
self.pre_calculated_mean = pre_calculated_mean
def __call__(self, tensor):
return tensor - self.pre_calculated_mean<create_dataframe>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
test = CTScansDataset(
 root_dir=test_dir,
transform=transforms.Compose([
CropBoundingBox() ,
ConvertToHU() ,
Resize(resize_dims),
Clip(bounds=clip_bounds),
MaskWatershed(
 min_hu=min(clip_bounds),
iterations=watershed_iterations,
show_tqdm=False),
Normalize(bounds=clip_bounds),
ToTensor()
])) 
<categorify>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
class CTTensorsDataset(Dataset):
def __init__(self, root_dir, transform=None):
self.root_dir = Path(root_dir)
 self.tensor_files = sorted([f for f in self.root_dir.glob('*.pt')])
 self.transform = transform
def __len__(self):
return len(self.tensor_files)
 def __getitem__(self, item):
if torch.is_tensor(item):
item = item.tolist()
image = torch.load(self.tensor_files[item])
 if self.transform:
image = self.transform(image)
 return {
'patient_id': self.tensor_files[item].stem,
'image': image
}
def mean(self):
cum = 0
for i in range(len(self)) :
sample = self[i]['image']
cum += torch.mean(sample ).item()
return cum / len(self)
 def random_split(self, val_size: float):
num_val = int(val_size * len(self))
num_train = len(self)- num_val
return random_split(self, [num_train, num_val] )<define_variables>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] )
train = CTTensorsDataset(
 root_dir='.. /input/osic-cached-dataset',
transform=ZeroCenter(pre_calculated_mean=pre_calculated_mean)
)
 cum = 0
for i in range(len(train)) :
sample = train[i]['image']
cum += torch.mean(sample ).item()
assert cum / len(train)== pytest.approx(0 )<categorify>
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
class AutoEncoder(nn.Module):
def __init__(self, latent_features=latent_features):
super(AutoEncoder, self ).__init__()
self.conv1 = nn.Conv3d(1, 16, 3)
 self.conv2 = nn.Conv3d(16, 32, 3)
 self.conv3 = nn.Conv3d(32, 96, 2)
 self.conv4 = nn.Conv3d(96, 1, 1)
 self.pool1 = nn.MaxPool3d(kernel_size=2, stride=2, return_indices=True)
 self.pool2 = nn.MaxPool3d(kernel_size=3, stride=3, return_indices=True)
 self.pool3 = nn.MaxPool3d(kernel_size=2, stride=2, return_indices=True)
 self.pool4 = nn.MaxPool3d(kernel_size=2, stride=2, return_indices=True)
 self.fc1 = nn.Linear(10 * 10, latent_features)
 self.fc2 = nn.Linear(latent_features, 10 * 10)
 self.deconv0 = nn.ConvTranspose3d(1, 96, 1)
 self.deconv1 = nn.ConvTranspose3d(96, 32, 2)
 self.deconv2 = nn.ConvTranspose3d(32, 16, 3)
 self.deconv3 = nn.ConvTranspose3d(16, 1, 3)
 self.unpool0 = nn.MaxUnpool3d(kernel_size=2, stride=2)
 self.unpool1 = nn.MaxUnpool3d(kernel_size=2, stride=2)
 self.unpool2 = nn.MaxUnpool3d(kernel_size=3, stride=3)
 self.unpool3 = nn.MaxUnpool3d(kernel_size=2, stride=2)
 def encode(self, x, return_partials=True):
x = self.conv1(x)
 up3out_shape = x.shape
x, i1 = self.pool1(x)
 x = self.conv2(x)
 up2out_shape = x.shape
x, i2 = self.pool2(x)
 x = self.conv3(x)
 up1out_shape = x.shape
x, i3 = self.pool3(x)
 x = self.conv4(x)
 up0out_shape = x.shape
x, i4 = self.pool4(x)
 x = x.view(-1, 10 * 10)
 x = F.relu(self.fc1(x))
if return_partials:
return x, up3out_shape, i1, up2out_shape, i2, up1out_shape, i3, \
up0out_shape, i4
else:
return x
def forward(self, x):
x, up3out_shape, i1, up2out_shape, i2, \
up1out_shape, i3, up0out_shape, i4 = self.encode(x)
 x = F.relu(self.fc2(x))
x = x.view(-1, 1, 1, 10, 10)
 x = self.unpool0(x, output_size=up0out_shape, indices=i4)
 x = self.deconv0(x)
 x = self.unpool1(x, output_size=up1out_shape, indices=i3)
 x = self.deconv1(x)
 x = self.unpool2(x, output_size=up2out_shape, indices=i2)
 x = self.deconv2(x)
 x = self.unpool3(x, output_size=up3out_shape, indices=i1)
 x = self.deconv3(x)
 return x<split>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
t0 = time()
data = CTTensorsDataset(
 root_dir='.. /input/osic-cached-dataset',
transform=ZeroCenter(pre_calculated_mean=pre_calculated_mean)
)
 train_set, val_set = data.random_split(val_size)
 datasets = {'train': train_set, 'val': val_set}
dataloaders = {
x: DataLoader(
 datasets[x],
batch_size=batch_size,
shuffle=(x == 'train'),
num_workers=2 
)for x in ['train', 'val']}
dataset_sizes = {x: len(datasets[x])for x in ['train', 'val']}
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = AutoEncoder(latent_features=latent_features ).to(device)
 criterion = torch.nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate)
 best_model_wts = None
best_loss = np.inf
date_time = datetime.now().strftime("%Y%m%d-%H%M")
 log_dir = Path(tensorboard_dir)/ f'{date_time}'
writer = SummaryWriter(log_dir )<categorify>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
latent_Dicom={}
for i in range(len(test)) :
Pid = test[i]['metadata'].PatientID
print(f'Inspecting CT Scan {Pid}')
 sample = test[i]['image'].squeeze(0 ).numpy()
with torch.no_grad() :
img = test[i]['image'].unsqueeze(0 ).float().to(device)
 latent_features = model.encode(img, return_partials=False)\ 
.squeeze().cpu().numpy().tolist()
latent_Dicom[Pid]=latent_features 
<categorify>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
for i in range(len(data)) :
Pid=data[i]["patient_id"]
sample = data[i]['image'].squeeze(0 ).numpy()
with torch.no_grad() :
img = data[i]['image'].unsqueeze(0 ).float().to(device)
 latent_features = model.encode(img, return_partials=False)\ 
.squeeze().cpu().numpy().tolist()
latent_Dicom[Pid]=latent_features 
<define_variables>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
latent_features_tot=3<concatenate>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
pl = All_data['Patient'].tolist()
pl2=[latent_Dicom[i] if i in latent_Dicom else np.nan for i in pl]
pl3=np.array(pl2)
 for i in range(latent_features_tot):
stri = 'latent_features'+str(i)
 pl4=pl3[:,i]
All_data[stri]=pl4
FE.append(stri )<split>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
tr = All_data.loc[All_data.WHERE=='train']
chunk = All_data.loc[All_data.WHERE=='val']
sub = All_data.loc[All_data.WHERE=='test']
tr.shape, chunk.shape, sub.shape<compute_test_metric>
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )