Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<feature_engineering>
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null
!python pytorch-xla-env-setup.py --version 1.6 --apt-packages libomp5 libopenblas-dev > /dev/null
!pip install transformers > /dev/null
!pip install pandarallel > /dev/null
for i in set(weird):
hhold=poverty_train[poverty_train['idhogar']==i][['idhogar', 'parentesco1', 'Target']]
target=hhold[hhold['parentesco1']==1]['Target'].tolist() [0]
for row in hhold.iterrows() :
idx=row[0]
if row[1]['parentesco1']!=1:
poverty_train.at[idx, 'Target']=target 
<filter>
tqdm.pandas()
nltk.download('punkt')
 pandarallel.initialize(nb_workers=2, progress_bar=True )
poverty_train[poverty_train['idhogar']==weird[1]][['idhogar','parentesco1', 'Target']]<filter>
SEED = 42
MAX_LENGTH = 224
BACKBONE_PATH = '.. /input/multitpu-inference'
CHECKPOINT_PATH = '.. /input/multitpu-inference/checkpoint-xlm-roberta.bin'
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
very_poor = poverty_train[poverty_train["Target"] == 1]
poor = poverty_train[poverty_train["Target"] == 2]
vulnerable = poverty_train[poverty_train["Target"] == 3]
safe = poverty_train[poverty_train["Target"] == 4]<compute_test_metric>
LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
def get_sentences(text, lang='en'):
return sent_tokenize(text, LANGS.get(lang, 'english'))
def exclude_duplicate_sentences(text, lang='en'):
sentences = []
for sentence in get_sentences(text, lang):
sentence = sentence.strip()
if sentence not in sentences:
sentences.append(sentence)
 return ' '.join(sentences)
 def clean_text(text, lang='en'):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 text = exclude_duplicate_sentences(text, lang)
 return text.strip()
def mean_confidence_interval(data, confidence=0.95):
a = 1.0*np.array(data)
 n = len(a)
 m, se = np.mean(a), scipy.stats.sem(a)
 h = se * sp.stats.t._ppf(( 1+confidence)/2., n-1)
 return m, m-h, m+h<compute_train_metric>
class DatasetRetriever(Dataset):
def __init__(self, df):
self.comment_texts = df['comment_text'].values
self.ids = df['id'].values
self.tokenizer = XLMRobertaTokenizer.from_pretrained(BACKBONE_PATH)
 def get_tokens(self, text):
encoded = self.tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=MAX_LENGTH,
pad_to_max_length=True 
)
 return encoded['input_ids'], encoded['attention_mask']
def __len__(self):
return self.ids.shape[0]
def __getitem__(self, idx):
text = self.comment_texts[idx]
tokens, attention_mask = self.get_tokens(text)
 tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)
 return self.ids[idx], tokens, attention_mask
display(mean_confidence_interval(very_poor["age"], confidence = 0.95),
mean_confidence_interval(safe["age"], confidence = 0.95))<feature_engineering>
%%time
df_test = pd.read_csv(f'.. /input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 df_test['comment_text'] = df_test.parallel_apply(lambda x: clean_text(x['content'], x['lang']), axis=1)
 df_test = df_test.drop(columns=['content'])
 df_test.head()
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop(['Id'], axis=1, inplace=True)
 df.drop(['hhsize', 'female', 'area2'], axis=1, inplace=True)
 return df
def convert_OHE2LE(df):
print_check = True
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
if print_check:
sum_ohe = df[cols_s_].sum(axis=1 ).unique()
if sum_ohe.shape[0]>1:
print(s_)
 print(df[cols_s_].sum(axis=1 ).value_counts())
 tmp_cat = df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df<categorify>
test_dataset = DatasetRetriever(df_test )
def process_df(df_):
encode_data(df_)
 for f_ in ['v2a1', 'v18q1', 'meaneduc', 'SQBmeaned']:
df_[f_] = df_[f_].fillna(0)
 df_['rez_esc'] = df_['rez_esc'].fillna(0)
 return do_features(df_)
 test = process_df(test)
<concatenate>
class ToxicSimpleNNModel(nn.Module):
def __init__(self, backbone):
super(ToxicSimpleNNModel, self ).__init__()
self.backbone = backbone
self.dropout = nn.Dropout(0.3)
 self.linear = nn.Linear(
 in_features=self.backbone.pooler.dense.out_features*2,
out_features=2, 
)
 def forward(self, input_ids, attention_masks):
bs, seq_length = input_ids.shape
seq_x, _ = self.backbone(input_ids=input_ids, attention_mask=attention_masks)
 apool = torch.mean(seq_x, 1)
 mpool, _ = torch.max(seq_x, 1)
 x = torch.cat(( apool, mpool), 1)
 x = self.dropout(x)
 return self.linear(x)
 backbone = XLMRobertaModel(XLMRobertaConfig.from_pretrained(BACKBONE_PATH))
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_<feature_engineering>
net = ToxicSimpleNNModel(backbone=backbone)
 checkpoint = torch.load(CHECKPOINT_PATH, map_location=torch.device('cpu'))
net.load_state_dict(checkpoint);
checkpoint = None
del checkpoint
train = do_features(poverty_train )<prepare_x_and_y>
def _mp_fn(rank, flags):
device = xm.xla_device()
model = net.to(device)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
pin_memory=False,
drop_last=False,
num_workers=1 
)
 fitter = MultiCoreTPUPredictor(model=model, device=device)
 fitter.run_inference(test_loader )
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1 )<define_variables>
ensemble = pd.read_csv('.. /input/multitpu-inference/submission-ensemble.csv', index_col='id')
 ensemble['toxic'].hist(bins=100);
cols_2_drop = ['agg18_estadocivil1_MEAN', 'agg18_estadocivil3_COUNT', 'agg18_estadocivil4_COUNT',
'agg18_estadocivil5_COUNT', 'agg18_estadocivil6_COUNT', 'agg18_estadocivil7_COUNT',
'agg18_instlevel1_COUNT', 'agg18_instlevel2_COUNT', 'agg18_instlevel3_COUNT',
'agg18_instlevel4_COUNT', 'agg18_instlevel5_COUNT', 'agg18_instlevel6_COUNT',
'agg18_instlevel7_COUNT', 'agg18_instlevel8_COUNT', 'agg18_instlevel9_COUNT',
'agg18_parentesco10_COUNT', 'agg18_parentesco10_MEAN', 'agg18_parentesco11_COUNT',
'agg18_parentesco11_MEAN', 'agg18_parentesco12_COUNT', 'agg18_parentesco12_MEAN',
'agg18_parentesco1_COUNT', 'agg18_parentesco2_COUNT', 'agg18_parentesco3_COUNT',
'agg18_parentesco4_COUNT', 'agg18_parentesco4_MEAN', 'agg18_parentesco5_COUNT',
'agg18_parentesco6_COUNT', 'agg18_parentesco6_MEAN', 'agg18_parentesco7_COUNT',
'agg18_parentesco7_MEAN', 'agg18_parentesco8_COUNT', 'agg18_parentesco8_MEAN',
'agg18_parentesco9_COUNT', 'fe_people_weird_stat', 'hacapo', 'hacdor', 'mobilephone',
'parentesco1', 'rez_esc', 'v14a', 'v18q',
'agg18_age_MIN', 'agg18_age_MAX' , 'agg18_age_MEAN', 'agg18_escolari_MIN',
'agg18_escolari_MAX', 'agg18_escolari_MEAN', 'agg18_dis_MEAN',
'agg18_estadocivil1_COUNT', 'agg18_estadocivil2_MEAN', 'agg18_estadocivil2_COUNT',
'agg18_estadocivil3_MEAN', 'agg18_estadocivil4_MEAN', 'agg18_estadocivil5_MEAN',
'agg18_estadocivil6_MEAN','agg18_estadocivil7_MEAN','agg18_parentesco1_MEAN',
'agg18_parentesco2_MEAN','agg18_parentesco3_MEAN','agg18_parentesco5_MEAN','agg18_parentesco9_MEAN',
'agg18_instlevel1_MEAN','agg18_instlevel2_MEAN','agg18_instlevel3_MEAN','agg18_instlevel4_MEAN',
'agg18_instlevel5_MEAN','agg18_instlevel6_MEAN','agg18_instlevel7_MEAN','agg18_instlevel8_MEAN',
'agg18_instlevel9_MEAN']
X.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True)
 test.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True )<normalization>
def scale_min_max_submission(submission):
min_, max_ = submission['toxic'].min() , submission['toxic'].max()
submission['toxic'] =(submission['toxic'] - min_)/(max_ - min_)
 return submission
<init_hyperparams><EOS> 
submission.to_csv('submission.csv' )