Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<data_type_conversions>
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null
!python pytorch-xla-env-setup.py --version 20200420 --apt-packages libomp5 libopenblas-dev > /dev/null
!pip install transformers==2.5.1 > /dev/null
!pip install pandarallel > /dev/null
!pip install catalyst==20.4.2 > /dev/null
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
os.environ['XLA_USE_BF16'] = "1"
tqdm.pandas()
nltk.download('punkt')
 pandarallel.initialize(nb_workers=4, progress_bar=False )
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
SEED = 42
MAX_LENGTH = 224
BACKBONE_PATH = 'xlm-roberta-large'
ROOT_PATH = f'.. '
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
def get_sentences(text, lang='en'):
return sent_tokenize(text, LANGS.get(lang, 'english'))
def exclude_duplicate_sentences(text, lang='en'):
sentences = []
for sentence in get_sentences(text, lang):
sentence = sentence.strip()
if sentence not in sentences:
sentences.append(sentence)
 return ' '.join(sentences)
 def clean_text(text, lang='en'):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 text = exclude_duplicate_sentences(text, lang)
 return text.strip()
class NLPTransform(BasicTransform):

 @property
def targets(self):
return {"data": self.apply}
def update_params(self, params, **kwargs):
if hasattr(self, "interpolation"):
params["interpolation"] = self.interpolation
if hasattr(self, "fill_value"):
params["fill_value"] = self.fill_value
return params
def get_sentences(self, text, lang='en'):
return sent_tokenize(text, LANGS.get(lang, 'english'))
class ShuffleSentencesTransform(NLPTransform):

 def __init__(self, always_apply=False, p=0.5):
super(ShuffleSentencesTransform, self ).__init__(always_apply, p)
 def apply(self, data, **params):
text, lang = data
sentences = self.get_sentences(text, lang)
 random.shuffle(sentences)
 return ' '.join(sentences), lang
class ExcludeDuplicateSentencesTransform(NLPTransform):

 def __init__(self, always_apply=False, p=0.5):
super(ExcludeDuplicateSentencesTransform, self ).__init__(always_apply, p)
 def apply(self, data, **params):
text, lang = data
sentences = []
for sentence in self.get_sentences(text, lang):
sentence = sentence.strip()
if sentence not in sentences:
sentences.append(sentence)
 return ' '.join(sentences), lang
class ExcludeNumbersTransform(NLPTransform):

 def __init__(self, always_apply=False, p=0.5):
super(ExcludeNumbersTransform, self ).__init__(always_apply, p)
 def apply(self, data, **params):
text, lang = data
text = re.sub(r'[0-9]', '', text)
 text = re.sub(r'\s+', ' ', text)
 return text, lang
class ExcludeHashtagsTransform(NLPTransform):
Exclude @users Exclude urls
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
class SynthesicOpenSubtitlesTransform(NLPTransform):
def __init__(self, always_apply=False, p=0.5):
super(SynthesicOpenSubtitlesTransform, self ).__init__(always_apply, p)
 df = pd.read_csv(f'{ROOT_PATH}/input/open-subtitles-toxic-pseudo-labeling/open-subtitles-synthesic.csv', index_col='id')[['comment_text', 'toxic', 'lang']]
df = df[~df['comment_text'].isna() ]
df['comment_text'] = df.parallel_apply(lambda x: clean_text(x['comment_text'], x['lang']), axis=1)
 df = df.drop_duplicates(subset='comment_text')
 df['toxic'] = df['toxic'].round().astype(np.int)
 self.synthesic_toxic = df[df['toxic'] == 1].comment_text.values
self.synthesic_non_toxic = df[df['toxic'] == 0].comment_text.values
del df
gc.collect() ;
def generate_synthesic_sample(self, text, toxic):
texts = [text]
if toxic == 0:
for i in range(random.randint(1,5)) :
texts.append(random.choice(self.synthesic_non_toxic))
else:
for i in range(random.randint(0,2)) :
texts.append(random.choice(self.synthesic_non_toxic))
for i in range(random.randint(1,3)) :
texts.append(random.choice(self.synthesic_toxic))
random.shuffle(texts)
 return ' '.join(texts)
 def apply(self, data, **params):
text, toxic = data
text = self.generate_synthesic_sample(text, toxic)
 return text, toxic
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
def get_train_transforms() :
return albumentations.Compose([
ExcludeUsersMentionedTransform(p=0.95),
ExcludeUrlsTransform(p=0.95),
ExcludeNumbersTransform(p=0.95),
ExcludeHashtagsTransform(p=0.95),
ExcludeDuplicateSentencesTransform(p=0.95),
], p=1.0)
 def get_synthesic_transforms() :
return SynthesicOpenSubtitlesTransform(p=0.5)
 train_transforms = get_train_transforms() ;
synthesic_transforms = get_synthesic_transforms()
tokenizer = XLMRobertaTokenizer.from_pretrained(BACKBONE_PATH)
 shuffle_transforms = ShuffleSentencesTransform(always_apply=True )
NFOLD = 50
kf = KFold(n_splits=NFOLD )<predict_on_test>
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, labels_or_ids, comment_texts, langs, use_train_transforms=False, test=False):
self.test = test
self.labels_or_ids = labels_or_ids
self.comment_texts = comment_texts
self.langs = langs
self.use_train_transforms = use_train_transforms
def get_tokens(self, text):
encoded = tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=MAX_LENGTH,
pad_to_max_length=True 
)
 return encoded['input_ids'], encoded['attention_mask']
def __len__(self):
return self.comment_texts.shape[0]
def __getitem__(self, idx):
text = self.comment_texts[idx]
lang = self.langs[idx]
if self.test is False:
label = self.labels_or_ids[idx]
target = onehot(2, label)
 if self.use_train_transforms:
text, _ = train_transforms(data=(text, lang)) ['data']
tokens, attention_mask = self.get_tokens(str(text))
token_length = sum(attention_mask)
 if token_length > 0.8*MAX_LENGTH:
text, _ = shuffle_transforms(data=(text, lang)) ['data']
elif token_length < 60:
text, _ = synthesic_transforms(data=(text, label)) ['data']
else:
tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)
 return target, tokens, attention_mask
tokens, attention_mask = self.get_tokens(str(text))
tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)
 if self.test is False:
return target, tokens, attention_mask
return self.labels_or_ids[idx], tokens, attention_mask
def get_labels(self):
return list(np.char.add(self.labels_or_ids.astype(str), self.langs))
%%time
cnt = 0
EPOCHS = 2000
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
%%time
df_train = pd.read_csv(f'{ROOT_PATH}/input/jigsaw-public-baseline-train-data/train_data.csv')
 train_dataset = DatasetRetriever(
 labels_or_ids=df_train['toxic'].values,
comment_texts=df_train['comment_text'].values,
langs=df_train['lang'].values,
use_train_transforms=True, 
)
 del df_train
gc.collect() ;
for targets, tokens, attention_masks in train_dataset:
break
print(targets)
 print(tokens.shape)
 print(attention_masks.shape )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
df_val = pd.read_csv(f'{ROOT_PATH}/input/jigsaw-multilingual-toxic-comment-classification/validation.csv', index_col='id')
 validation_tune_dataset = DatasetRetriever(
 labels_or_ids=df_val['toxic'].values,
comment_texts=df_val['comment_text'].values,
langs=df_val['lang'].values,
use_train_transforms=True, 
)
 df_val['comment_text'] = df_val.parallel_apply(lambda x: clean_text(x['comment_text'], x['lang']), axis=1)
 validation_dataset = DatasetRetriever(
 labels_or_ids=df_val['toxic'].values,
comment_texts=df_val['comment_text'].values,
langs=df_val['lang'].values,
use_train_transforms=False, 
)
 del df_val
gc.collect() ;
for targets, tokens, attention_masks in validation_dataset:
break
print(targets)
 print(tokens.shape)
 print(attention_masks.shape )
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )<feature_engineering>
df_test = pd.read_csv(f'{ROOT_PATH}/input/jigsaw-multilingual-toxic-comment-classification/test.csv', index_col='id')
 df_test['comment_text'] = df_test.parallel_apply(lambda x: clean_text(x['content'], x['lang']), axis=1)
 test_dataset = DatasetRetriever(
 labels_or_ids=df_test.index.values,
comment_texts=df_test['comment_text'].values,
langs=df_test['lang'].values,
use_train_transforms=False,
test=True 
)
 del df_test
gc.collect() ;
for ids, tokens, attention_masks in test_dataset:
break
print(ids)
 print(tokens.shape)
 print(attention_masks.shape )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1)
 y_pred = nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,1]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = sklearn.metrics.roc_auc_score(self.y_true, self.y_pred, labels=np.array([0, 1]))
@property
def avg(self):
return self.score
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
class LabelSmoothing(nn.Module):
def __init__(self, smoothing = 0.1):
super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
def forward(self, x, target):
if self.training:
x = x.float()
target = target.float()
logprobs = torch.nn.functional.log_softmax(x, dim = -1)
 nll_loss = -logprobs * target
nll_loss = nll_loss.sum(-1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
else:
return torch.nn.functional.cross_entropy(x, target )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )<prepare_output>
warnings.filterwarnings("ignore")
 class TPUFitter:
def __init__(self, model, device, config):
if not os.path.exists('node_submissions'):
os.makedirs('node_submissions')
 self.config = config
self.epoch = 0
self.log_path = 'log.txt'
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = AdamW(optimizer_grouped_parameters, lr=config.lr*xm.xrt_world_size())
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.criterion = config.criterion
xm.master_print(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
para_loader = pl.ParallelLoader(train_loader, [self.device])
 losses, final_scores = self.train_one_epoch(para_loader.per_device_loader(self.device))
self.log(f'[RESULT]: Train.Epoch: {self.epoch}, loss: {losses.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 t = time.time()
para_loader = pl.ParallelLoader(validation_loader, [self.device])
 losses, final_scores = self.validation(para_loader.per_device_loader(self.device))
self.log(f'[RESULT]: Validation.Epoch: {self.epoch}, loss: {losses.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=final_scores.avg)
 self.epoch += 1
def run_tuning_and_inference(self, test_loader, validation_tune_loader):
for e in range(2):
self.optimizer.param_groups[0]['lr'] = self.config.lr*xm.xrt_world_size() /(e + 1)
 para_loader = pl.ParallelLoader(validation_tune_loader, [self.device])
 losses, final_scores = self.train_one_epoch(para_loader.per_device_loader(self.device))
para_loader = pl.ParallelLoader(test_loader, [self.device])
 self.run_inference(para_loader.per_device_loader(self.device))
def validation(self, val_loader):
self.model.eval()
losses = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(targets, inputs, attention_masks)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
xm.master_print(
 f'Valid Step {step}, loss: ' + \
f'{losses.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}' 
)
 with torch.no_grad() :
inputs = inputs.to(self.device, dtype=torch.long)
 attention_masks = attention_masks.to(self.device, dtype=torch.long)
 targets = targets.to(self.device, dtype=torch.float)
 outputs = self.model(inputs, attention_masks)
 loss = self.criterion(outputs, targets)
 batch_size = inputs.size(0)
 final_scores.update(targets, outputs)
 losses.update(loss.detach().item() , batch_size)
 return losses, final_scores
def train_one_epoch(self, train_loader):
self.model.train()
losses = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(targets, inputs, attention_masks)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
self.log(
 f'Train Step {step}, loss: ' + \
f'{losses.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}' 
)
 inputs = inputs.to(self.device, dtype=torch.long)
 attention_masks = attention_masks.to(self.device, dtype=torch.long)
 targets = targets.to(self.device, dtype=torch.float)
 self.optimizer.zero_grad()
outputs = self.model(inputs, attention_masks)
 loss = self.criterion(outputs, targets)
 batch_size = inputs.size(0)
 final_scores.update(targets, outputs)
 losses.update(loss.detach().item() , batch_size)
 loss.backward()
xm.optimizer_step(self.optimizer)
 if self.config.step_scheduler:
self.scheduler.step()
self.model.eval()
self.save('last-checkpoint.bin')
 return losses, final_scores
def run_inference(self, test_loader):
self.model.eval()
result = {'id': [], 'toxic': []}
t = time.time()
for step,(ids, inputs, attention_masks)in enumerate(test_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
xm.master_print(f'Prediction Step {step}, time: {(time.time() - t):.5f}')
 with torch.no_grad() :
inputs = inputs.to(self.device, dtype=torch.long)
 attention_masks = attention_masks.to(self.device, dtype=torch.long)
 outputs = self.model(inputs, attention_masks)
 toxics = nn.functional.softmax(outputs, dim=1 ).data.cpu().numpy() [:,1]
result['id'].extend(ids.cpu().numpy())
 result['toxic'].extend(toxics)
 result = pd.DataFrame(result)
 node_count = len(glob('node_submissions/*.csv'))
result.to_csv(f'node_submissions/submission_{node_count}_{datetime.utcnow().microsecond}_{random.random() }.csv', index=False)
 def save(self, path):
xm.save(self.model.state_dict() , path)
 def log(self, message):
if self.config.verbose:
xm.master_print(message)
 with open(self.log_path, 'a+')as logger:
xm.master_print(f'{message}', logger )
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
class ToxicSimpleNNModel(nn.Module):
def __init__(self):
super(ToxicSimpleNNModel, self ).__init__()
self.backbone = XLMRobertaModel.from_pretrained(BACKBONE_PATH)
 self.dropout = nn.Dropout(0.3)
 self.linear = nn.Linear(
 in_features=self.backbone.pooler.dense.out_features*2,
out_features=2, 
)
 def forward(self, input_ids, attention_masks):
bs, seq_length = input_ids.shape
seq_x, _ = self.backbone(input_ids=input_ids, attention_mask=attention_masks)
 apool = torch.mean(seq_x, 1)
 mpool, _ = torch.max(seq_x, 1)
 x = torch.cat(( apool, mpool), 1)
 x = self.dropout(x)
 return self.linear(x )
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
net = ToxicSimpleNNModel()
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.26*df1['Confidence'] + 0.74*df2['Confidence']
df.head()<save_to_csv>
class TrainGlobalConfig:
num_workers = 0
batch_size = 16
n_epochs = 3
lr = 0.5 * 1e-5
verbose = True
verbose_step = 50
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='max',
factor=0.7,
patience=0,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08 
)
 criterion = LabelSmoothing()
df.to_csv('submission.csv', index=False )<import_modules>
def _mp_fn(rank, flags):
device = xm.xla_device()
net.to(device)
 train_sampler = DistributedSamplerWrapper(
 sampler=BalanceClassSampler(labels=train_dataset.get_labels() , mode="downsampling"),
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True 
)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=train_sampler,
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers, 
)
 validation_sampler = torch.utils.data.distributed.DistributedSampler(
 validation_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 validation_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=validation_sampler,
pin_memory=False,
drop_last=False,
num_workers=TrainGlobalConfig.num_workers 
)
 validation_tune_sampler = torch.utils.data.distributed.DistributedSampler(
 validation_tune_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True 
)
 validation_tune_loader = torch.utils.data.DataLoader(
 validation_tune_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=validation_tune_sampler,
pin_memory=False,
drop_last=False,
num_workers=TrainGlobalConfig.num_workers 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=test_sampler,
pin_memory=False,
drop_last=False,
num_workers=TrainGlobalConfig.num_workers 
)
 if rank == 0:
time.sleep(1)
 fitter = TPUFitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, validation_loader)
 fitter.run_tuning_and_inference(test_loader, validation_tune_loader )
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import cv2
import time
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<load_from_csv>
file = open('.. /input/jigsaw-public-baseline-results/log.txt', 'r')
 for line in file.readlines() :
print(line[:-1])
 file.close()
def seed_everything(seed=1000000007):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 print(time.time())
 seed_everything(int(time.time()))
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
All_data = tr.append([chunk, sub] )<feature_engineering>
submission = pd.read_csv('.. /input/jigsaw-public-baseline-results/submission.csv', index_col='id')
 submission.hist(bins=100 )
<drop_column><EOS> 
submission.to_csv('submission.csv' )