Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<predict_on_test>
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null
!python pytorch-xla-env-setup.py --version 20200416 --apt-packages libomp5 libopenblas-dev > /dev/null
!pip install transformers > /dev/null
!pip install pandarallel > /dev/null
for test, sample in tqdm.tqdm(env.iter_test()):
test = preprocess(test, test=True)
 test_tmp = test.copy()
df, _ = transform(test_tmp, player_features, game_features, test=True)
 pred = make_pred(df, sample, env, models)
 pred = post_process(pred, test)
 pred = np.cumsum(pred, axis=1)
 pred = pd.DataFrame(data=pred,columns=sample.columns)
 pred = pred.clip(0, 1)
 env.predict(pred)
 env.write_submission_file()<set_options>
tqdm.pandas()
nltk.download('punkt')
 pandarallel.initialize(nb_workers=2, progress_bar=True )
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )<import_modules>
SEED = 42
MAX_LENGTH = 224
BACKBONE_PATH = '.. /input/multitpu-inference'
CHECKPOINT_PATH = '.. /input/multitpu-inference/checkpoint-xlm-roberta.bin'
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
from keras.layers import Dense
from keras.models import Sequential
from keras.callbacks import Callback, EarlyStopping
from keras.layers import Dropout, PReLU, BatchNormalization, ELU, GaussianNoise, Activation
from keras.optimizers import Adam<load_from_csv>
LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
def get_sentences(text, lang='en'):
return sent_tokenize(text, LANGS.get(lang, 'english'))
def exclude_duplicate_sentences(text, lang='en'):
sentences = []
for sentence in get_sentences(text, lang):
sentence = sentence.strip()
if sentence not in sentences:
sentences.append(sentence)
 return ' '.join(sentences)
 def clean_text(text, lang='en'):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 text = exclude_duplicate_sentences(text, lang)
 return text.strip()
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', usecols=['GameId', 'PlayId', 'Team', 'X', 'Y', 'S', 'A', 'Dis',
'Orientation', 'Dir', 'NflId', 'DisplayName', 'YardLine',
'Quarter', 'GameClock', 'PossessionTeam', 'Down', 'Distance',
'FieldPosition', 'NflIdRusher', 'OffenseFormation',
'OffensePersonnel', 'DefendersInTheBox', 'DefensePersonnel',
'PlayDirection', 'TimeHandoff', 'TimeSnap', 'Yards',
'WindSpeed', 'Temperature', 'Humidity'] )<remove_duplicates>
class DatasetRetriever(Dataset):
def __init__(self, df):
self.comment_texts = df['comment_text'].values
self.ids = df['id'].values
self.tokenizer = XLMRobertaTokenizer.from_pretrained(BACKBONE_PATH)
 def get_tokens(self, text):
encoded = self.tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=MAX_LENGTH,
pad_to_max_length=True 
)
 return encoded['input_ids'], encoded['attention_mask']
def __len__(self):
return self.ids.shape[0]
def __getitem__(self, idx):
text = self.comment_texts[idx]
tokens, attention_mask = self.get_tokens(text)
 tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)
 return self.ids[idx], tokens, attention_mask
outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()<prepare_x_and_y>
%%time
df_test = pd.read_csv(f'.. /input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 df_test['comment_text'] = df_test.parallel_apply(lambda x: clean_text(x['content'], x['lang']), axis=1)
 df_test = df_test.drop(columns=['content'])
 df_test.head()
def create_features(df, deploy=False):
def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0
def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')
 return df
def projection_features(df):
rad = 2 * np.pi *(90 - df[['Orientation']])/ 360
v_0 = df['S'].values * np.cos(rad ).values.reshape(-1)
 v_1 = np.sin(rad ).values.reshape(-1)
 a_0 = df['A'].values * np.cos(rad ).values.reshape(-1)
 a_1 = np.sin(rad)
 df['v_0'] = v_0
df['v_1'] = v_1
df['a_0'] = a_0
df['a_1'] = a_1
return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
def static_features(df):
static_features = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','X','Y','S','A','Dis','Orientation','Dir',
'YardLine','Quarter','Down','Distance','DefendersInTheBox']].drop_duplicates()
static_features['DefendersInTheBox'] = static_features['DefendersInTheBox'].fillna(np.mean(static_features['DefendersInTheBox']))
return static_features
def combine_features(relative_to_back, defense, static, speed_frame, deploy=deploy):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df, speed_frame, on=['GameId','PlayId'],how='inner')
 if not deploy:
df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')
 return df
df = projection_features(df)
 speed_frame= df[['GameId','PlayId',
'a_0','a_1','v_0','v_1', 'Temperature', 'Humidity']]
yardline = update_yardline(df)
 df = update_orientation(df, yardline)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 static_feats = static_features(df)
 basetable = combine_features(rel_back, def_feats, static_feats, speed_frame, deploy=deploy)
 basetable[['a_0','a_1','v_0','v_1']] = basetable[['a_0','a_1','v_0','v_1']].fillna(0)
 basetable['Temperature'] = basetable['Temperature'].fillna(60.436442)
 basetable['Humidity'] = basetable['Humidity'].fillna(59.0)
 return basetable<feature_engineering>
test_dataset = DatasetRetriever(df_test )
%time train_basetable = create_features(train, False )<prepare_x_and_y>
class ToxicSimpleNNModel(nn.Module):
def __init__(self, backbone):
super(ToxicSimpleNNModel, self ).__init__()
self.backbone = backbone
self.dropout = nn.Dropout(0.3)
 self.linear = nn.Linear(
 in_features=self.backbone.pooler.dense.out_features*2,
out_features=2, 
)
 def forward(self, input_ids, attention_masks):
bs, seq_length = input_ids.shape
seq_x, _ = self.backbone(input_ids=input_ids, attention_mask=attention_masks)
 apool = torch.mean(seq_x, 1)
 mpool, _ = torch.max(seq_x, 1)
 x = torch.cat(( apool, mpool), 1)
 x = self.dropout(x)
 return self.linear(x)
 backbone = XLMRobertaModel(XLMRobertaConfig.from_pretrained(BACKBONE_PATH))
X = train_basetable.copy()
yards = X.Yards
y = np.zeros(( yards.shape[0], 1))
_y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx] = target
_y[idx][99 + target] = 1
X.drop(['GameId', 'PlayId', 'Yards'], axis=1, inplace=True)
 scaler = StandardScaler()
X = scaler.fit_transform(X )<choose_model_class>
net = ToxicSimpleNNModel(backbone=backbone)
 checkpoint = torch.load(CHECKPOINT_PATH, map_location=torch.device('cpu'))
net.load_state_dict(checkpoint);
checkpoint = None
del checkpoint
def get_model(size=X.shape[1]):
model = Sequential()
model.add(Dense(256, input_dim=size, activation=None))
model.add(PReLU())
 model.add(Dropout(0.5))
model.add(Dense(256, activation=None))
model.add(PReLU())
 model.add(BatchNormalization())
 model.add(Dense(1, activation=None))
model.compile(optimizer=Adam(lr=0.001), loss='mae')
 return model
def get_model_v2(size=X.shape[1]):
model = Sequential()
model.add(Dense(256, input_dim=size, activation=None))
model.add(BatchNormalization())
 model.add(PReLU())
 model.add(Dropout(0.5))
model.add(Dense(256, activation=None))
model.add(BatchNormalization())
 model.add(PReLU())
 model.add(BatchNormalization())
 model.add(Dense(199, activation='softmax'))
model.compile(optimizer=Adam(lr=0.001), loss='categorical_crossentropy', metrics=[])
 return model<import_modules>
def _mp_fn(rank, flags):
device = xm.xla_device()
model = net.to(device)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
pin_memory=False,
drop_last=False,
num_workers=1 
)
 fitter = MultiTPUPredictor(model=model, device=device)
 fitter.run_inference(test_loader )
from sklearn.model_selection import KFold, GroupKFold<prepare_x_and_y>
ensemble = pd.read_csv('.. /input/multitpu-inference/submission-ensemble.csv', index_col='id')
 ensemble['toxic'].hist(bins=100 )
models = []
oof_prediction = np.zeros(shape=(X.shape[0]))
rkf = GroupKFold(6)
 FOLD_LIST = list(rkf.split(train_basetable['GameId'], train_basetable['GameId'], train_basetable['GameId']))
for fold_id,(tr_idx, vl_idx)in enumerate(FOLD_LIST):
print(fold_id)
 x_tr, y_tr = X[tr_idx], y[tr_idx]
x_vl, y_vl = X[vl_idx], y[vl_idx]
model = get_model(X.shape[1])
 es = EarlyStopping(monitor = 'val_loss',
mode = 'min',
restore_best_weights = True,
verbose = 1,
patience = 5)
 model.fit(x_tr, y_tr, epochs=200, batch_size=1024, validation_data=[x_vl, y_vl], callbacks=[es])
 oof_prediction[vl_idx] += model.predict(x_vl ).reshape(-1)
 models.append(model)
<import_modules>
def scale_min_max_submission(submission):
min_, max_ = submission['toxic'].min() , submission['toxic'].max()
submission['toxic'] =(submission['toxic'] - min_)/(max_ - min_)
 return submission
<compute_test_metric><EOS> 
submission.to_csv('submission.csv' )