Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<find_best_params>
!pip install torch > /dev/null
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null
!python pytorch-xla-env-setup.py --version 20200515 --apt-packages libomp5 libopenblas-dev > /dev/null
!pip install transformers > /dev/null
!pip install pandarallel > /dev/null
!pip install emoji
!pip install num2words
!pip install --upgrade language_tool_python
!pip install --upgrade turkishnlp
!pip install --upgrade git+https://github.com/pytorch/contrib.git > /dev/null
cvres = rnd_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(mean_score, params )<find_best_params>
os.environ['XLA_USE_BF16'] = "1"
pandarallel.initialize(nb_workers=8, progress_bar=False)
print(rnd_search.best_score_, rnd_search.best_estimator_, rnd_search.best_params_ )<train_model>
MODEL = 'xlm-roberta-large'
root_dir = Path(".. /input")
 test_file_dir = Path(root_dir, "jigsaw-multilingual-toxic-comment-classification")
 model_dir = Path(root_dir, MODEL)
 checkpoint_dir =(model_dir/'best_model.bin')
 config_path =(model_dir/'config.json')
 out_dir = Path('.. /output')
 test_file = "test.csv"
test_eng_file1 = "jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv"
test_eng_file2 = "test-en-df/test_en.csv"
sub_file = "jigsaw-toxic/sample_submission.csv"
open_subtitles_file = 'open-subtitles-toxic/open-subtitles-synthesic.csv'
LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
with open(config_path)as f:
model_config = json.load(f)
 MODEL_VERSION = model_config['model_version']
MODEL_PREFIX = model_config['model_prefix']
MAX_LENGTH = model_config['max_len']
english_pipeline = model_config['english pipeline']
output_hidden_states = model_config['output_hidden_states']
<save_to_csv>
dir_paths = {'base_dir': test_file_dir, 'base_t_dir': root_dir, 'test_file':test_file,
'test_file_translated1': test_eng_file1, 'test_file_translated2': test_eng_file2}
_, _, test, _ = utility.read_data(dir_paths, list(LANGS.keys()), english_pipeline=english_pipeline)
 if english_pipeline:
test_yandex, test_google = test[0], test[1]
test = test_yandex.append(test_google)
 input_cols_test = ['content']
df_test = clean.clean_data(test, input_cols_test, True)
 test.head()
y_subm = pd.read_csv('.. /input/sample_submission.csv')
 y_subm['Target'] = rnd_search.predict(test)+ 1
y_subm.to_csv('submission.csv', index=False )<load_from_csv>
tokenizer = AutoTokenizer.from_pretrained(str(model_dir))
data=pd.read_csv('.. /input/train.csv' )<filter>
test_dataset = models.DatasetRetriever(
 tokenizer,
labels_or_ids=test.index.values,
comment_texts=test[input_cols_test].values,
langs=test['lang'].values,
maxlen=MAX_LENGTH,
use_train_transforms=False,
test=True 
)
 del test
gc.collect() ;
data[(data['dependency']=='no')&(data['SQBdependency']!=0)]<filter>
transformer = AutoModel.from_config(AutoConfig.from_pretrained(str(model_dir)))
 net = models.ToxicSimpleNNModel(transformer, config.TrainGlobalConfig)
 checkpoint = torch.load(checkpoint_dir, map_location=torch.device('cpu'))
net.load_state_dict(checkpoint);
config.TrainGlobalConfig.train_lenght = len(test_dataset)
 checkpoint = None
del checkpoint
data[(data['dependency']=='yes')&(data['SQBdependency']!=1)]<filter>
warnings.filterwarnings("ignore")
 def _mp_fn(rank, flags):
device = xm.xla_device()
net.to(device)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
pin_memory=False,
drop_last=False,
num_workers=1 
)
 fitter = models.TPUFitter(model=net, device=device, config=config.TrainGlobalConfig,
base_model_path=model_dir, model_name=MODEL, model_prefix=MODEL_PREFIX,
model_version=MODEL_VERSION,
out_path=out_dir)
 fitter.run_inference(test_loader )
data[(data['dependency']=='3')&(data['SQBdependency']!=9)]<feature_engineering>
submission = pd.concat([pd.read_csv(path)for path in(out_dir/'node_submissions' ).glob('*.csv')] ).groupby('id' ).mean()
submission['toxic'].hist(bins=100 )
<filter><EOS> 
submission.to_csv('submission.csv' )