Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<define_variables>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.cluster_spec().as_dict() ['worker'])
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
preds =(0.7*lasso_preds)+(0.3*xgb_preds )<save_to_csv>
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev
solution = pd.DataFrame({"id":test_set.Id, "SalePrice":preds})
 solution.to_csv("housePrice_solution.csv", index = False )<set_options>
os.environ['XLA_USE_BF16'] = "1"
os.environ['XLA_TENSOR_ALLOCATOR_MAXSIZE'] = '100000000'
warnings.filterwarnings("ignore" )
pd.set_option('max_columns', 50 )<split>
PATH = ""
MAX_LEN = 224
env = twosigmanews.make_env()
market_train_df, news_train = env.get_training_data()<feature_engineering>
df_train = pd.read_csv(".. /input/jigsaw-public-baseline-train-data/train_data.csv", usecols=["comment_text", "toxic", "lang"])
 df_valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv', usecols=["comment_text", "toxic", "lang"])
 df_test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv' ).rename(columns={"content": "comment_text"} )
start = datetime(2009, 1, 1, 0, 0, 0 ).date()
market_train = market_train_df.loc[market_train_df['time'].dt.date >= start].reset_index(drop=True)
 news_train = news_train.loc[news_train['time'].dt.date >= start].reset_index(drop=True )<drop_column>
labels = np.char.add(df_train.toxic.values.astype(str), df_train.lang.values)
 df_train["label"] = labels
min_size = df_train.groupby("label" ).size().min()
print(min_size)
 upsample = 1
samples = []
for i in range(3):
print(i)
 sample = []
for l in df_train.label.unique() :
if l[0] == "1":
x = df_train[df_train["label"]==l].sample(min_size, replace=False, random_state=i)
 sample.append(x)
 sample.append(df_train[df_train["label"]==f"0{l[1:]}"].sample(min_size*upsample, replace=False, random_state=i))
sample = pd.concat(sample, axis=0 ).sample(frac=1)
 del sample["label"]
DATA_LENGTH = len(sample)
 samples.append(sample)
 del sample
def preprocess_news(news_train):
drop_list = [
'audiences', 'subjects', 'assetName',
'headline', 'firstCreated', 'sourceTimestamp',
]
news_train.drop(drop_list, axis=1, inplace=True)
 for col in ['headlineTag', 'provider', 'sourceId']:
news_train[col], uniques = pd.factorize(news_train[col])
 del uniques
news_train['assetCodes'] = news_train['assetCodes'].apply(lambda x: x[1:-1].replace("'", ""))
return news_train<string_transform>
del df_train
df_train = samples
news_train = preprocess_news(news_train )<drop_column>
gc.collect()
!free -h
def unstack_asset_codes(news_train):
codes = []
indexes = []
for i, values in news_train['assetCodes'].iteritems() :
explode = values.split(", ")
 codes.extend(explode)
 repeat_index = [int(i)]*len(explode)
 indexes.extend(repeat_index)
 index_df = pd.DataFrame({'news_index': indexes, 'assetCode': codes})
 del codes, indexes
gc.collect()
return index_df<drop_column>
class CustomRoberta(nn.Module):
def __init__(self):
super(CustomRoberta, self ).__init__()
self.num_labels = 2
self.roberta = transformers.XLMRobertaModel.from_pretrained("xlm-roberta-large", output_hidden_states=False, num_labels=1)
 self.dropout = nn.Dropout(p=0.2)
 self.ln = nn.LayerNorm(1024)
 self.classifier = nn.Linear(1024, self.num_labels)
 def forward(self,
input_ids=None,
attention_mask=None,
position_ids=None,
head_mask=None,
inputs_embeds=None):
o1, o2 = self.roberta(input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds)
 x1 = torch.mean(o1, 1)
 x = x1
x = self.ln(x)
 x = self.dropout(x)
 logits = self.classifier(x)
 return logits
index_df = unstack_asset_codes(news_train)
 index_df.head()<merge>
tokenizer = transformers.XLMRobertaTokenizer.from_pretrained('xlm-roberta-large')
 mx = xmp.MpModelWrapper(CustomRoberta() )
def merge_news_on_index(news_train, index_df):
news_train['news_index'] = news_train.index.copy()
news_unstack = index_df.merge(news_train, how='left', on='news_index')
 news_unstack.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return news_unstack<merge>
class BERTDataset:
def __init__(self, df=None):
self.df = df
def __len__(self):
return len(self.df)
 def get_tokens(self, text):
encoded = tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=MAX_LEN,
pad_to_max_length=True 
)
 return encoded['input_ids'], encoded['attention_mask']
def get_labels(self):
return LabelEncoder().fit_transform(np.char.add(self.df.toxic.values.astype(str), self.df.lang.values ).reshape(-1,1)).astype(np.int16)
 def __getitem__(self, item):
text = self.df.iloc[item]["comment_text"]
lang = self.df.iloc[item]["lang"]
encoded = self.get_tokens(text)
 targets = np.zeros(2)
 if "toxic" in self.df.columns:
targets[self.df.iloc[item]["toxic"]] = 1
return {
'ids': torch.tensor(encoded[0]),
'mask': torch.tensor(encoded[1]),
'targets': targets,
'index': item
}
news_unstack = merge_news_on_index(news_train, index_df)
 del news_train, index_df
gc.collect()
news_unstack.head(3 )<data_type_conversions>
valid_dataset = BERTDataset(df_valid)
 test_dataset = BERTDataset(df_test )
def group_news(news_frame):
news_frame['date'] = news_frame.time.dt.date
aggregations = ['mean']
gp = news_frame.groupby(['assetCode', 'date'] ).agg(aggregations)
 gp.columns = pd.Index(["{}_{}".format(e[0], e[1])for e in gp.columns.tolist() ])
 gp.reset_index(inplace=True)
 float_cols = {c: 'float32' for c in gp.columns if c not in ['assetCode', 'date']}
return gp.astype(float_cols )<groupby>
gc.collect()
!free -h
news_agg = group_news(news_unstack)
 del news_unstack; gc.collect()
news_agg.head(3 )<feature_engineering>
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1)
 y_pred = nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,1]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = metrics.roc_auc_score(self.y_true, self.y_pred, labels=np.array([0, 1]))
@property
def avg(self):
return self.score
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def train_loop_fn(data_loader, model, optimizer, device, scheduler=None, epoch=None):
model.train()
losses = AverageMeter()
auc = RocAucMeter()
start_time = time.time()
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids=ids,
attention_mask = mask, 
)
 loss = loss_fn(outputs, targets)
 loss.backward()
xm.optimizer_step(optimizer)
 loss = loss.detach().item()
auc.update(targets, outputs)
 losses.update(loss, ids.size(0))
if bi % 10 == 0:
xm.master_print(f'bi={bi}, loss={losses.avg:<8.4f}, auc={auc.avg:<8.4f} {time.time() -start_time:<2.2f}')
 if scheduler is not None:
scheduler.step()
del loss
del losses
del outputs
del ids
del targets
gc.collect()
model.eval()
def eval_loop_fn(data_loader, model, device):
fin_targets = []
fin_outputs = []
fin_index = []
with torch.no_grad() :
for bi, d in enumerate(data_loader):
if bi % 10 == 0:
xm.master_print(f'EVAL bi={bi}')
 ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
index = d["index"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids = ids,
attention_mask = mask, 
)
 targets_np = targets.cpu().detach().numpy().argmax(axis=1 ).tolist()
outputs_np = outputs.cpu().detach().numpy() [:,1].tolist()
fin_targets.extend(targets_np)
 fin_outputs.extend(outputs_np)
 fin_index.extend(index.tolist())
 return fin_outputs, fin_targets, fin_index
def process_date(df):
df['date'] = df['time'].dt.date
df['month'] = df['time'].dt.month
df['dayofweek'] = df['time'].dt.dayofweek
return df<feature_engineering>
class SmoothLoss(nn.Module):
def __init__(self):
super(SmoothLoss, self ).__init__()
def forward(self, pred, target):
pred = pred.log_softmax(dim=1)
 return torch.mean(torch.sum(-target * pred, dim=1))
def loss_fn(outputs, targets):
return SmoothLoss()(outputs, targets)
 TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 32
EPOCHS = 1
LR = 2e-5
def _run() :
gc.collect()
xm.master_print('starting run')
 valid_sampler = torch.utils.data.distributed.DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=VALID_BATCH_SIZE,
sampler=valid_sampler,
drop_last=False,
num_workers=0 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=VALID_BATCH_SIZE,
sampler=test_sampler,
drop_last=False,
num_workers=0 
)
 gc.collect()
device = xm.xla_device()
model = mx.to(device)
 xm.master_print('done loading model')
 num_train_steps = int(len(df_train[0])/ TRAIN_BATCH_SIZE / xm.xrt_world_size())
 optimizer = AdamW([{'params': model.roberta.parameters() , 'lr': LR},
{'params': [param for name, param in model.named_parameters() if 'roberta' not in name], 'lr': 1e-3} ], lr=LR, weight_decay=0)
 scheduler = get_cosine_schedule_with_warmup(
 optimizer,
num_warmup_steps = 0,
num_training_steps = num_train_steps * EPOCHS 
)
 xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size() }')
 for epoch in range(EPOCHS):
train_dataset = BERTDataset(df_train[epoch])
 train_sampler = torch.utils.data.distributed.DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=0,
shuffle=False 
)
 train_sampler.set_epoch(epoch)
 para_loader = pl.ParallelLoader(train_data_loader, [device])
 xm.master_print('parallel loader created...training now')
 train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler, epoch=epoch)
 del para_loader
gc.collect()
if epoch == EPOCHS-1:
xm.master_print('saving model')
 xser.save(model.state_dict() , f"{PATH}model.bin", master_only=True)
 xm.master_print('model saved')
 para_loader = pl.ParallelLoader(valid_data_loader, [device])
 o, t, i = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 auc = metrics.roc_auc_score(np.array(t), o)
 gc.collect()
del para_loader
print(f'[xla:{xm.get_ordinal() }] AUC = {auc}')
 def reduce_fn(vals):
return sum(vals)/ len(vals)
 auc = xm.mesh_reduce('auc_reduce', auc, reduce_fn)
 xm.master_print(f'AUC AVG = {auc}')
 para_loader = pl.ParallelLoader(test_data_loader, [device])
 o, t, i = eval_loop_fn(para_loader.per_device_loader(device), model, device)
 del t
gc.collect()
return o, i
def process_ma(df,columns=['open','close','volume'], windows=[5, 20, 60]):
ma_columns = []
ma_dev_columns = []
for col in columns:
for window in windows:
ma_column = 'ma_{0}_{1}'.format(col,window)
 ma_dev_column = 'ma_dev_{0}_{1}'.format(col,window)
 std_column = 'std_{0}_{1}'.format(col,window)
 ma_columns.append(ma_column)
 ma_dev_columns.append(ma_dev_column)
 df[ma_column] = df.groupby('assetCode')[col].apply(lambda x: x.rolling(window ).mean())
 df[ma_dev_column] = df[col] / df[ma_column] - 1
df[std_column] = df.groupby('assetCode')[col].apply(lambda x: x.rolling(window ).std())
 return df<categorify>
test_preds = np.zeros(len(df_test))
for i in range(8):
test_preds[np.load(f"test_index_{i}.npy", allow_pickle=True ).reshape(-1)] = np.load(f"test_preds_{i}.npy", allow_pickle=True ).reshape(-1 )
market_train = process_date(market_train)
 market_train = process_ma(market_train)
<merge>
sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub['toxic'] = test_preds
df = market_train.merge(news_agg, how='left', on=['assetCode', 'date'])
 del market_train, news_agg
gc.collect()
df.head(3 )<compute_test_metric>
sub.to_csv('submission.csv', index=False )
<data_type_conversions><EOS> 
sub.to_csv('submission.csv', index=False )