Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<set_options>
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev
data_augmentation_layers = tf.keras.Sequential(
 [
layers.experimental.preprocessing.RandomCrop(height=image_size, width=image_size),
layers.experimental.preprocessing.RandomFlip("horizontal_and_vertical"),
layers.experimental.preprocessing.RandomRotation(0.25),
layers.experimental.preprocessing.RandomZoom(( -0.2, 0)) ,
layers.experimental.preprocessing.RandomContrast(( 0.2,0.2))
]
)<train_model>
warnings.filterwarnings("ignore" )
image = Image.open(".. /input/cassava-leaf-disease-classification/train_images/3412658650.jpg")
 plt.imshow(image)
 plt.show()<concatenate>
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch import optim
from torch.utils.data import DataLoader
import torchvision
from torchvision import transforms
from tqdm.notebook import tqdm
image = tf.expand_dims(np.array(image), 0 )<normalization>
input = torch.randn(10,device=device)
 linear = torch.nn.Linear(10, 20 ).to(device)
 output = linear(input)
 print(output )
plt.figure(figsize=(10, 10))
for i in range(9):
augmented_image = data_augmentation_layers(image)
 ax = plt.subplot(3, 3, i + 1)
 plt.imshow(augmented_image[0])
 plt.axis("off" )<choose_model_class>
transform = transforms.ToTensor()
dataset = torchvision.datasets.MNIST(root='./data', train=True,
download=True, transform=transform)
 train_loader = DataLoader(dataset,batch_size=256)
 class MNIST(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1)
 self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1)
 self.conv3 = nn.Conv2d(16, 10, kernel_size=3, stride=2, padding=1)
 def forward(self, xb):
xb = xb.view(-1, 1, 28, 28)
 xb = F.relu(self.conv1(xb))
xb = F.relu(self.conv2(xb))
xb = F.relu(self.conv3(xb))
xb = F.avg_pool2d(xb, 4)
 return xb.view(-1, xb.size(1))
def train_loop() :
model = MNIST().train().to(device)
 loss_fn = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters() , lr=1e-2, momentum=0.9)
 for data, target in tqdm(train_loader,total=len(train_loader)) :
optimizer.zero_grad()
data = data.to(device)
 target = target.to(device)
 output = model(data)
 loss = loss_fn(output, target)
 loss.backward()
optimizer.step()
print(f'Final Loss: {loss}' )
efficientnet = EfficientNetB3(weights=".. /input/efficientnetb3-notop/efficientnetb3_notop.h5",
include_top=False,
input_shape=input_shape,
drop_connect_rate=dropout_rate)
 inputs = Input(shape=input_shape)
 augmented = data_augmentation_layers(inputs)
 efficientnet = efficientnet(augmented)
 pooling = layers.GlobalAveragePooling2D()(efficientnet)
 dropout = layers.Dropout(dropout_rate )(pooling)
 outputs = Dense(len(classes_to_predict), activation="softmax" )(dropout)
 model = Model(inputs=inputs, outputs=outputs)
 model.summary()<load_pretrained>
os.environ['XLA_USE_BF16']="1"
os.environ['XLA_TENSOR_ALLOCATOR_MAXSIZE'] = '100000000'
%%time
model.get_layer('efficientnetb3' ).get_layer('normalization' ).adapt(adapt_data_batches )<choose_model_class>
class AverageMeter:

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
epochs = 8
decay_steps = int(round(len(training_df)/batch_size)) *epochs
cosine_decay = CosineDecay(initial_learning_rate=1e-4, decay_steps=decay_steps, alpha=0.3)
 callbacks = [ModelCheckpoint(filepath='best_model.h5', monitor='val_loss', save_best_only=True)]
model.compile(loss="sparse_categorical_crossentropy", optimizer=tf.keras.optimizers.Adam(cosine_decay), metrics=["accuracy"] )<train_model>
class ArrayDataset(torch.utils.data.Dataset):
def __init__(self,*arrays):
assert all(arrays[0].shape[0] == array.shape[0] for array in arrays)
 self.arrays = arrays
def __getitem__(self, index):
return tuple(torch.from_numpy(np.array(array[index])) for array in self.arrays)
 def __len__(self):
return self.arrays[0].shape[0]
history = model.fit(training_data_batches,
epochs = epochs,
validation_data=validation_data_batches,
callbacks=callbacks )<load_pretrained>
class CustomRoberta(nn.Module):
def __init__(self):
super(CustomRoberta, self ).__init__()
self.num_labels = 1
self.hid_mix = 3
self.roberta = transformers.XLMRobertaModel.from_pretrained("xlm-roberta-large", output_hidden_states=True, num_labels=1)
 self.dropout = nn.Dropout(p=0.2)
 self.feats = self.roberta.pooler.dense.out_features
self.classifier = nn.Linear(self.feats, self.num_labels)
 def forward(self,
input_ids=None,
attention_mask=None,
position_ids=None,
head_mask=None,
inputs_embeds=None):
outputs = self.roberta(input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds)
 hidden_states = outputs[2]
hmix = []
for i in range(1, self.hid_mix + 1):
hmix.append(hidden_states[-i][:, 0].reshape(( -1, 1, self.feats)))
 hmix_tensor = torch.cat(hmix, 1)
 mean_tensor = torch.mean(hmix_tensor, 1)
 pool_tensor = self.dropout(mean_tensor)
 return self.classifier(pool_tensor )
model.load_weights("best_model.h5" )<define_search_model>
mx =CustomRoberta()
test_time_augmentation_layers = tf.keras.Sequential(
 [
layers.experimental.preprocessing.RandomFlip("horizontal_and_vertical"),
layers.experimental.preprocessing.RandomZoom(( -0.2, 0)) ,
layers.experimental.preprocessing.RandomContrast(( 0.2,0.2))
]
)<predict_on_test>
tokenized_path = '.. /input/xlm-r-large-tokenize-xhlulu-dataset/'
def run_predictions_over_image_list(image_list, folder):
predictions = []
with tqdm(total=len(image_list)) as pbar:
for image_filename in image_list:
pbar.update(1)
 predictions.append(predict_and_vote(image_filename, folder))
return predictions<predict_on_test>
x_train = np.load(tokenized_path+'x_train.npy',mmap_mode='r')
 train_toxic = np.load(tokenized_path+'df_train_toxic.npy',mmap_mode='r')
 x_valid = np.load(tokenized_path+'x_valid.npy',mmap_mode='r')
 valid_toxic = np.load(tokenized_path+'df_valid_toxic.npy',mmap_mode='r' )
validation_df["results"] = run_predictions_over_image_list(validation_df["image_id"], training_folder )<define_variables>
train_dataset = ArrayDataset(x_train, train_toxic)
 valid_dataset = ArrayDataset(x_valid, valid_toxic )
true_positives = 0
prediction_distribution_per_class = {"0":{"0": 0, "1": 0, "2":0, "3":0, "4":0},
"1":{"0": 0, "1": 0, "2":0, "3":0, "4":0},
"2":{"0": 0, "1": 0, "2":0, "3":0, "4":0},
"3":{"0": 0, "1": 0, "2":0, "3":0, "4":0},
"4":{"0": 0, "1": 0, "2":0, "3":0, "4":0}}
number_of_images = len(validation_df)
 for idx, pred in validation_df.iterrows() :
if int(pred["label"])== pred.results:
true_positives+=1
prediction_distribution_per_class[str(pred["label"])][str(pred.results)] += 1
print("accuracy: {}%".format(true_positives/number_of_images*100))<feature_engineering>
del x_train, x_valid
test_folder = '.. /input/cassava-leaf-disease-classification/test_images/'
submission_df = pd.DataFrame(columns={"image_id","label"})
 submission_df["image_id"] = os.listdir(test_folder)
 submission_df["label"] = 0<predict_on_test>
gc.collect()
submission_df["label"] = run_predictions_over_image_list(submission_df["image_id"], test_folder )<save_to_csv>
def loss_fn(outputs, targets):
return torch.nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
submission_df.to_csv("submission.csv", index=False )<set_options>
def reduce_fn(vals):
return sum(vals)/ len(vals )
%matplotlib inline
warnings.filterwarnings("ignore" )<categorify>
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
for bi, d in enumerate(data_loader):
ids = d[0]
targets = d[1]
ids = ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids, 
)
 loss = loss_fn(outputs, targets)
 if bi % 50 == 0:
loss_reduced = xm.mesh_reduce('loss_reduce',loss,reduce_fn)
 xm.master_print(f'bi={bi}, loss={loss_reduced}')
 loss.backward()
xm.optimizer_step(optimizer)
 if scheduler is not None:
scheduler.step()
model.eval()
def eval_loop_fn(data_loader, model, device):
fin_targets = []
fin_outputs = []
for bi, d in enumerate(data_loader):
ids = d[0]
targets = d[1]
ids = ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids=ids, 
)
 targets_np = targets.cpu().detach().numpy().tolist()
outputs_np = outputs.cpu().detach().numpy().tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 del targets_np, outputs_np
gc.collect()
return fin_outputs, fin_targets
def encode_data(df):
df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'])
 def feature_importance(forest, X_train):
ranked_list = []
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
print("Feature ranking:")
 for f in range(X_train.shape[1]):
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 return ranked_list<feature_engineering>
def _run() :
MAX_LEN = 192
BATCH_SIZE = 16
EPOCHS = 2
train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=0, 
)
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=BATCH_SIZE,
sampler=valid_sampler,
drop_last=False,
num_workers=0 
)
 device = xm.xla_device()
model = mx.to(device)
 xm.master_print('done loading model')
 param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
xm.master_print('training on train dataset')
 lr = 0.5e-5 * xm.xrt_world_size()
num_train_steps = int(len(train_dataset)/ BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 xm.master_print(f'num_training_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 for epoch in range(EPOCHS):
gc.collect()
para_loader = pl.ParallelLoader(train_data_loader, [device])
 xm.master_print('parallel loader created...training now')
 gc.collect()
train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 del para_loader
para_loader = pl.ParallelLoader(valid_data_loader, [device])
 gc.collect()
o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 del para_loader
gc.collect()
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 auc_reduced = xm.mesh_reduce('auc_reduce',auc,reduce_fn)
 xm.master_print(f'AUC = {auc_reduced}')
 gc.collect()
xm.master_print('training on validation set')
 lr = 1.5e-5 * xm.xrt_world_size()
num_train_steps = int(len(valid_dataset)/ BATCH_SIZE / xm.xrt_world_size() * EPOCHS)
 optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0.1*num_train_steps,
num_training_steps=num_train_steps 
)
 xm.master_print(f'num_training_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 for epoch in range(EPOCHS):
gc.collect()
para_loader = pl.ParallelLoader(valid_data_loader, [device])
 xm.master_print('parallel loader created...training now')
 gc.collect()
train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)
 del para_loader
para_loader = pl.ParallelLoader(valid_data_loader, [device])
 gc.collect()
o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 del para_loader
gc.collect()
auc = metrics.roc_auc_score(np.array(t)>= 0.5, o)
 auc_reduced = xm.mesh_reduce('auc_reduce',auc,reduce_fn)
 xm.master_print(f'AUC = {auc_reduced}')
 gc.collect()
xm.save(model.state_dict() , "xlm_roberta_model.bin")
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop(['Id'], axis=1, inplace=True)
 return df<categorify>
def _mp_fn(rank, flags):
a = _run()
FLAGS={}
start_time = time.time()
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df<load_from_csv>
print('Time taken for training: ',time.time() -start_time )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ids = test.Id<categorify>
gc.collect()
def process_df(df_):
encode_data(df_)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )<feature_engineering>
x_test = np.load(tokenized_path+'x_test.npy',mmap_mode='r')
 test_ids = np.load(tokenized_path+'test_df_ids.npy',mmap_mode='r' )
train['dependency'] = np.sqrt(train['SQBdependency'])
 test['dependency'] = np.sqrt(test['SQBdependency'])
 train.loc[train['edjefa'] == "no", "edjefa"] = 0
train.loc[train['edjefe'] == "no", "edjefe"] = 0
test.loc[test['edjefa'] == "no", "edjefa"] = 0
test.loc[test['edjefe'] == "no", "edjefe"] = 0
train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "edjefa"] = train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "escolari"]
train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "edjefe"] = train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "edjefa"] = test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "edjefe"] = test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "escolari"]
train.loc[train['edjefa'] == "yes", "edjefa"] = 4
train.loc[train['edjefe'] == "yes", "edjefe"] = 4
test.loc[test['edjefa'] == "yes", "edjefa"] = 4
test.loc[test['edjefe'] == "yes", "edjefe"] = 4
train['edjefe'] = train['edjefe'].astype("int")
 train['edjefa'] = train['edjefa'].astype("int")
 test['edjefe'] = test['edjefe'].astype("int")
 test['edjefa'] = test['edjefa'].astype("int")
 train['edjef'] = np.max(train[['edjefa','edjefe']], axis=1)
 test['edjef'] = np.max(test[['edjefa','edjefe']], axis=1)
 train['v2a1']=train['v2a1'].fillna(0)
 test['v2a1']=test['v2a1'].fillna(0)
 test['v18q1']=test['v18q1'].fillna(0)
 train['v18q1']=train['v18q1'].fillna(0)
 train['rez_esc']=train['rez_esc'].fillna(0)
 test['rez_esc']=test['rez_esc'].fillna(0)
 train.loc[train.meaneduc.isnull() , "meaneduc"] = 0
train.loc[train.SQBmeaned.isnull() , "SQBmeaned"] = 0
test.loc[test.meaneduc.isnull() , "meaneduc"] = 0
test.loc[test.SQBmeaned.isnull() , "SQBmeaned"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "v14a"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "sanitario1"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "v14a"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "sanitario1"] = 0<concatenate>
test_dataset = ArrayDataset(test_ids,x_test )
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_<categorify>
trained_state_dict = torch.load("xlm_roberta_model.bin")
 mx.load_state_dict(trained_state_dict)
 del trained_state_dict
gc.collect()
train, test = train_test_apply_func(train, test, convert_OHE2LE )<categorify>
def gather_submissions() :
ids = []
outputs = []
for core in range(8):
ids.extend(np.load(f'ids_core{core}.npy'))
outputs.extend(np.load(f'outputs_core{core}.npy'))
submission_df = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 print(len(ids))
submission_df['toxic'].iloc[ids] = np.array(outputs)[:,0]
submission_df.to_csv('submission.csv',index=False )
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )<categorify>
def test_loop_fn(data_loader, model, device):
fin_id = []
fin_outputs = []
for bi, d in enumerate(data_loader):
text_id = d[0]
ids = d[1]
ids = ids.to(device, dtype=torch.long)
 with torch.no_grad() :
outputs = model(
 input_ids=ids, 
)
 outputs_cpu = outputs.data.cpu()
fin_id.append(text_id)
 fin_outputs.append(outputs_cpu)
 del text_id, ids, outputs_cpu
gc.collect()
xm.master_print('done with inference, saving...')
 fin_id_cat = torch.cat(fin_id)
 fin_outputs_cat = torch.cat(fin_outputs)
 np.save(f'ids_core{xm.get_ordinal() }.npy',fin_id_cat.cpu().numpy())
 np.save(f'outputs_core{xm.get_ordinal() }.npy',fin_outputs_cat.numpy())
train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0)
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
df['dependency_yes'] = df['dependency'].apply(lambda x: 1 if x == 'yes' else 0)
 df['dependency_no'] = df['dependency'].apply(lambda x: 1 if x == 'no' else 0)
 extract_features(train)
 extract_features(test )<drop_column>
def _run_test() :
BATCH_SIZE=64
test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
drop_last=False,
num_workers=0, 
)
 device = xm.xla_device()
model = mx.to(device)
 xm.master_print('done loading model')
 para_loader = pl.ParallelLoader(test_data_loader, [device])
 test_loop_fn(para_loader.per_device_loader(device),model,device)
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'female', ]
instlevel_cols = [s for s in train.columns.tolist() if 'instlevel' in s]
needless_cols.extend(instlevel_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )<split>
gc.collect()
<split><EOS> 
sub = pd.read_csv('submission.csv')
 test = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub.loc[test["lang"] == "es", "toxic"] *= 1.06
sub.loc[test["lang"] == "fr", "toxic"] *= 1.04
sub.loc[test["lang"] == "it", "toxic"] *= 0.97
sub.loc[test["lang"] == "pt", "toxic"] *= 0.96
sub.loc[test["lang"] == "tr", "toxic"] *= 0.98
sub.toxic -= sub.toxic.min()
sub.toxic /= sub.toxic.max()
sub.toxic.hist(bins=100, log=True, alpha=0.6)
 sub.to_csv('submission.csv', index=False )