Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<merge>
import codecs
import copy
import csv
import gc
import os
import pickle
import random
import tempfile
import time
from typing import Dict, List, Sequence, Set, Tuple, Union
uxp = uxp.merge(orders_last, on='user_id', how='left')
 uxp.head(10 )<merge>
import matplotlib.pyplot as plt
from matplotlib import figure
import numpy as np
from scipy.stats import hmean
import seaborn as sns
from sklearn.manifold import TSNE
from sklearn.metrics import roc_auc_score, roc_curve
from skopt import gp_minimize
from skopt.plots import plot_convergence, plot_objective, plot_evaluations
from skopt.space import Real, Integer
from skopt.utils import use_named_args
import tensorflow as tf
import tensorflow_probability as tfp
from tensorflow.python.framework import ops, tensor_util
from tensorflow.python.keras.utils import losses_utils, tf_utils
from tensorflow.python.ops import math_ops
from tensorflow.python.ops.losses import util as tf_losses_util
import tensorflow_addons as tfa
from transformers import AutoTokenizer, XLMRobertaTokenizer
from transformers import TFXLMRobertaModel, XLMRobertaConfig
uxp_train = uxp[uxp.eval_set=='train']
uxp_train = uxp_train.merge(order_products_train, on=['product_id', 'order_id'], how='left')
 uxp_train = uxp_train.drop(['order_id','eval_set', 'add_to_cart_order'], axis=1)
 uxp_train = uxp_train.fillna(0)
 uxp_train.head(20 )<drop_column>
class LossFunctionWrapper(tf.keras.losses.Loss):
def __init__(self,
fn,
reduction=losses_utils.ReductionV2.AUTO,
name=None,
**kwargs):
super(LossFunctionWrapper, self ).__init__(reduction=reduction, name=name)
 self.fn = fn
self._fn_kwargs = kwargs
def call(self, y_true, y_pred):
if tensor_util.is_tensor(y_pred)and tensor_util.is_tensor(y_true):
y_pred, y_true = tf_losses_util.squeeze_or_expand_dimensions(y_pred, y_true)
 return self.fn(y_true, y_pred, **self._fn_kwargs)
 def get_config(self):
config = {}
for k, v in six.iteritems(self._fn_kwargs):
config[k] = tf.keras.backend.eval(v)if tf_utils.is_tensor_or_variable(v)\
else v
base_config = super(LossFunctionWrapper, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
uxp_test = uxp[uxp.eval_set=='test']
uxp_test = uxp_test.drop(['eval_set', 'order_id'], axis=1)
 uxp_test = uxp_test.fillna(0)
 uxp_test.head(20 )<set_options>
def distance_based_log_loss(y_true, y_pred):
y_pred = ops.convert_to_tensor(y_pred)
 y_true = math_ops.cast(y_true, y_pred.dtype)
 margin = 1.0
p =(1.0 + tf.math.exp(-margin)) /(1.0 + tf.math.exp(y_pred - margin))
return tf.keras.backend.binary_crossentropy(target=y_true, output=p )
del uxp
del orders_last
gc.collect()<split>
class DBLLogLoss(LossFunctionWrapper):
def __init__(self, reduction=losses_utils.ReductionV2.AUTO,
name='distance_based_log_loss'):
super(DBLLogLoss, self ).__init__(distance_based_log_loss, name=name,
reduction=reduction )
uxp_train.loc[:, 'reordered'] = uxp_train.reordered.fillna(0)
 X_train, X_val, y_train, y_val = train_test_split(uxp_train.drop('reordered', axis=1), uxp_train.reordered,
test_size=0.2, random_state=42)

gc.collect()
d_train = xgboost.DMatrix(X_train, y_train)
 param = {'max_depth':10,
'eta':0.02,
'colsample_bytree':0.4,
'subsample':0.75,
'silent':1,
'nthread':27,
'eval_metric':'logloss',
'binary':'logistic',
'tree_method':'hist'
}
watchlist= [(d_train, "train")]
bst = xgboost.train(params=param, dtrain=d_train, num_boost_round=1000, evals=watchlist, early_stopping_rounds=40, verbose_eval=5)
 xgboost.plot_importance(bst )<drop_column>
class MaskCalculator(tf.keras.layers.Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(MaskCalculator, self ).__init__(**kwargs)
 def build(self, input_shape):
super(MaskCalculator, self ).build(input_shape)
 def call(self, inputs, **kwargs):
return tf.keras.backend.permute_dimensions(
 x=tf.keras.backend.repeat(
 x=tf.keras.backend.cast(
 x=tf.keras.backend.greater(
 x=inputs,
y=0 
),
dtype='float32' 
),
n=self.output_dim 
),
pattern=(0, 2, 1)
)
 def compute_output_shape(self, input_shape):
assert len(input_shape)== 1
shape = list(input_shape)
 shape.append(self.output_dim)
 return tuple(shape )
del [X_train, X_val, y_train, y_val]
gc.collect()<predict_on_test>
class WeightPosteriorCallback(tf.keras.callbacks.Callback):
def __init__(self, log_every: int):
self.bayesian_layers = []
self.layer_names = []
self.logged_epochs = dict()
self.log_every = log_every
super(WeightPosteriorCallback, self ).__init__()
def on_train_begin(self, logs=None):
self.bayesian_layers = list(filter(
 lambda layer:(layer.name.lower().startswith('bayesianhiddenlayer')or \
layer.name.lower().startswith('bayesianoutputlayer')) ,
self.model.layers 
))
self.layer_names = []
max_layer_idx = 0
for layer in self.bayesian_layers:
if layer.name.lower().startswith('bayesianhiddenlayer'):
new_layer_name = 'Layer'
start_pos = len('bayesianhiddenlayer')
 find_idx = layer.name[start_pos:].find('_')
 assert find_idx > 0
layer_idx = int(layer.name[start_pos:(start_pos + find_idx)])
 new_layer_name += '{0}'.format(layer_idx)
 if layer_idx > max_layer_idx:
max_layer_idx = layer_idx
else:
new_layer_name = 'Layer{0}'.format(max_layer_idx + 1)
 self.layer_names.append(new_layer_name)
 self.logged_epochs = dict()
def on_epoch_end(self, epoch, logs=None):
if(epoch == 0)or(((epoch + 1)% self.log_every)== 0):
qm_vals = [layer.kernel_posterior.mean() for layer in self.bayesian_layers]
qs_vals = [layer.kernel_posterior.stddev() for layer in self.bayesian_layers]
self.logged_epochs[epoch + 1] =(qm_vals, qs_vals )
d_test = xgboost.DMatrix(uxp_test)
 uxp_test = uxp_test.reset_index()
uxp_test = uxp_test[['product_id', 'user_id']]
uxp_test["reordered"] = bst.predict(d_test)
 del bst<filter>
def regular_encode(texts: List[str], tokenizer: XLMRobertaTokenizer,
maxlen: int)-> Tuple[np.ndarray, np.ndarray]:
err_msg = '"{0}" is wrong type for the text list!'.format(type(texts))
assert isinstance(texts, list)or isinstance(texts, tuple), err_msg
enc_di = tokenizer.batch_encode_plus(
 texts,
return_token_type_ids=False,
padding='max_length',
max_length=maxlen 
)
 err_msg = '{0} != {1}'.format(len(texts), len(enc_di['input_ids']))
assert len(texts)== len(enc_di['input_ids']), err_msg
err_msg = '{0} != {1}'.format(len(texts), len(enc_di['attention_mask']))
assert len(texts)== len(enc_di['attention_mask']), err_msg
encoded_tokens = np.zeros(( len(texts), maxlen), dtype=np.int32)
 encoded_masks = np.zeros(( len(texts), maxlen), dtype=np.int32)
 for sample_idx,(encoded_cur_text, encoded_cur_mask)in enumerate(
 zip(enc_di['input_ids'], enc_di['attention_mask'])
):
n_text = len(encoded_cur_text)
 n_mask = len(encoded_cur_mask)
 err_msg = 'Tokens and masks of texts "{0}" are different! '\
'{1} != {2}'.format(texts[sample_idx], n_text, n_mask)
 assert n_text == n_mask, err_msg
if n_text >= maxlen:
encoded_tokens[sample_idx] = np.array(encoded_cur_text[0:maxlen],
dtype=np.int32)
 encoded_masks[sample_idx] = np.array(encoded_cur_mask[0:maxlen],
dtype=np.int32)
 else:
padding = [0 for _ in range(maxlen - n_text)]
encoded_tokens[sample_idx] = np.array(encoded_cur_text + padding,
dtype=np.int32)
 encoded_masks[sample_idx] = np.array(encoded_cur_mask + padding,
dtype=np.int32)
 return encoded_tokens, encoded_masks
orders_test = orders[orders.eval_set=='test']<merge>
def load_train_set(file_name: str, text_field: str, sentiment_fields: List[str],
lang_field: str)-> Dict[str, List[Tuple[str, int]]]:
assert len(sentiment_fields)> 0, 'List of sentiment fields is empty!'
header = []
line_idx = 1
data_by_lang = dict()
with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore')as fp:
data_reader = csv.reader(fp, quotechar='"', delimiter=',')
 for row in data_reader:
if len(row)> 0:
err_msg = 'File "{0}": line {1} is wrong!'.format(file_name, line_idx)
 if len(header)== 0:
header = copy.copy(row)
 err_msg2 = err_msg + ' Field "{0}" is not found!'.format(text_field)
 assert text_field in header, err_msg2
for cur_field in sentiment_fields:
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(
 cur_field)
 assert cur_field in header, err_msg2
text_field_index = header.index(text_field)
 try:
lang_field_index = header.index(lang_field)
 except:
lang_field_index = -1
indices_of_sentiment_fields = []
for cur_field in sentiment_fields:
indices_of_sentiment_fields.append(header.index(cur_field))
else:
if len(row)== len(header):
text = row[text_field_index].strip()
assert len(text)> 0, err_msg + ' Text is empty!'
if lang_field_index >= 0:
cur_lang = row[lang_field_index].strip()
assert len(cur_lang)> 0, err_msg + ' Language is empty!'
else:
cur_lang = 'en'
max_proba = 0.0
for cur_field_idx in indices_of_sentiment_fields:
try:
cur_proba = float(row[cur_field_idx])
 except:
cur_proba = -1.0
err_msg2 = err_msg + ' Value {0} is wrong!'.format(
 row[cur_field_idx] 
)
 assert(cur_proba >= 0.0)and(cur_proba <= 1.0), err_msg2
if cur_proba > max_proba:
max_proba = cur_proba
new_label = 1 if max_proba >= 0.5 else 0
if cur_lang not in data_by_lang:
data_by_lang[cur_lang] = []
data_by_lang[cur_lang].append(( text, new_label))
if line_idx % 10000 == 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx,
file_name))
line_idx += 1
if line_idx > 0:
if(line_idx - 1)% 10000 != 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx - 1,
file_name))
return data_by_lang
uxp_test = uxp_test.merge(orders_test[["user_id", "order_id"]], on='user_id', how='left' ).drop('user_id', axis=1)
 uxp_test.columns = ['product_id', 'prediction', 'order_id']
uxp_test.product_id = uxp_test.product_id.astype(int)
 uxp_test.order_id = uxp_test.order_id.astype(int)
 uxp_test.head()<set_options>
def load_test_set(file_name: str, id_field: str, text_field: str,
lang_field: str)-> Dict[str, List[Tuple[str, int]]]:
header = []
line_idx = 1
data_by_lang = dict()
with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore')as fp:
data_reader = csv.reader(fp, quotechar='"', delimiter=',')
 for row in data_reader:
if len(row)> 0:
err_msg = 'File "{0}": line {1} is wrong!'.format(file_name, line_idx)
 if len(header)== 0:
header = copy.copy(row)
 err_msg2 = err_msg + ' Field "{0}" is not found!'.format(text_field)
 assert text_field in header, err_msg2
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(id_field)
 assert id_field in header, err_msg2
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(lang_field)
 assert lang_field in header, err_msg2
id_field_index = header.index(id_field)
 text_field_index = header.index(text_field)
 lang_field_index = header.index(lang_field)
 else:
if len(row)== len(header):
try:
id_value = int(row[id_field_index])
 except:
id_value = -1
err_msg2 = err_msg + ' {0} is wrong ID!'.format(
 row[id_field_index])
 assert id_value >= 0, err_msg2
text = row[text_field_index].strip()
assert len(text)> 0, err_msg + ' Text is empty!'
if lang_field_index >= 0:
cur_lang = row[lang_field_index].strip()
assert len(cur_lang)> 0, err_msg + ' Language is empty!'
else:
cur_lang = 'en'
if cur_lang not in data_by_lang:
data_by_lang[cur_lang] = []
data_by_lang[cur_lang].append(( text, id_value))
if line_idx % 10000 == 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx,
file_name))
line_idx += 1
if line_idx > 0:
if(line_idx - 1)% 10000 != 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx - 1,
file_name))
return data_by_lang
del orders
del orders_test
gc.collect()<find_best_params>
def build_siamese_dataset(texts: Dict[str, List[Tuple[str, int]]],
dataset_size: int, tokenizer: XLMRobertaTokenizer,
maxlen: int, batch_size: int,
shuffle: bool)-> Tuple[tf.data.Dataset, int]:
language_pairs = set()
for language in texts.keys() :
for other_language in texts:
if other_language == language:
language_pairs.add(( language, other_language))
else:
new_pair =(language, other_language)
 new_pair_2 =(other_language, language)
 if(new_pair not in language_pairs)and(new_pair_2 not in language_pairs):
language_pairs.add(new_pair)
 language_pairs = sorted(list(language_pairs))
print('Possible language pairs are: {0}.'.format(language_pairs))
err_msg = '{0} is too small size of the data set!'.format(dataset_size)
 assert dataset_size >=(len(language_pairs)* 10), err_msg
n_samples_for_lang_pair = int(np.ceil(dataset_size / float(len(language_pairs))))
text_pairs_and_labels = []
for left_lang, right_lang in language_pairs:
print('{0}-{1}:'.format(left_lang, right_lang))
left_positive_indices = list(filter(
 lambda idx: texts[left_lang][idx][1] > 0, range(len(texts[left_lang])) 
))
left_negative_indices = list(filter(
 lambda idx: texts[left_lang][idx][1] == 0, range(len(texts[left_lang])) 
))
right_positive_indices = list(filter(
 lambda idx: texts[right_lang][idx][1] > 0, range(len(texts[right_lang])) 
))
right_negative_indices = list(filter(
 lambda idx: texts[right_lang][idx][1] == 0, range(len(texts[right_lang])) 
))
used_pairs = set()
number_of_samples = 0
for _ in range(n_samples_for_lang_pair // 4):
left_idx = random.choice(left_positive_indices)
 right_idx = random.choice(right_positive_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_positive_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
1 
)
)
 number_of_samples += 1
print(' number of "1-1" pairs is {0};'.format(number_of_samples))
number_of_samples = 0
for _ in range(n_samples_for_lang_pair // 4,(2 * n_samples_for_lang_pair)// 4):
left_idx = random.choice(left_negative_indices)
 right_idx = random.choice(right_negative_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_negative_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
1 
)
)
 number_of_samples += 1
print(' number of "0-0" pairs is {0};'.format(number_of_samples))
number_of_samples = 0
for _ in range(( 2 * n_samples_for_lang_pair)// 4, n_samples_for_lang_pair):
left_idx = random.choice(left_negative_indices)
 right_idx = random.choice(right_positive_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_positive_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
if random.random() >= 0.5:
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
0 
)
)
 else:
text_pairs_and_labels.append(
(
 texts[right_lang][right_idx][0],
texts[left_lang][left_idx][0],
0 
)
)
 number_of_samples += 1
print(' number of "0-1" or "1-0" pairs is {0}.'.format(number_of_samples))
random.shuffle(text_pairs_and_labels)
 n_steps = len(text_pairs_and_labels)// batch_size
print('Samples number of the data set is {0}.'.format(len(text_pairs_and_labels)))
 print('Samples number per each language pair is {0}.'.format(n_samples_for_lang_pair))
tokens_of_left_texts, mask_of_left_texts = regular_encode(
 texts=[cur[0] for cur in text_pairs_and_labels],
tokenizer=tokenizer, maxlen=maxlen 
)
 tokens_of_right_texts, mask_of_right_texts = regular_encode(
 texts=[cur[1] for cur in text_pairs_and_labels],
tokenizer=tokenizer, maxlen=maxlen 
)
 siamese_labels = np.array([cur[2] for cur in text_pairs_and_labels], dtype=np.int32)
 print('Number of positive siamese samples is {0} from {1}.'.format(
 int(sum(siamese_labels)) , siamese_labels.shape[0]))
if shuffle:
err_msg = '{0} is too small number of samples for the data set!'.format(
 len(text_pairs_and_labels))
assert n_steps >= 50, err_msg
dataset = tf.data.Dataset.from_tensor_slices(
(
(
 tokens_of_left_texts, mask_of_left_texts,
tokens_of_right_texts, mask_of_right_texts 
),
siamese_labels 
)
 ).repeat().batch(batch_size)
 else:
dataset = tf.data.Dataset.from_tensor_slices(
(
(
 tokens_of_left_texts, mask_of_left_texts,
tokens_of_right_texts, mask_of_right_texts 
),
siamese_labels 
)
 ).batch(batch_size)
 del text_pairs_and_labels
return dataset, n_steps
class F1Optimizer() :
def __init__(self):
pass
@staticmethod
def get_expectations(P, pNone=None):
expectations = []
P = np.sort(P)[::-1]
n = np.array(P ).shape[0]
DP_C = np.zeros(( n + 2, n + 1))
if pNone is None:
pNone =(1.0 - P ).prod()
DP_C[0][0] = 1.0
for j in range(1, n):
DP_C[0][j] =(1.0 - P[j - 1])* DP_C[0, j - 1]
for i in range(1, n + 1):
DP_C[i, i] = DP_C[i - 1, i - 1] * P[i - 1]
for j in range(i + 1, n + 1):
DP_C[i, j] = P[j - 1] * DP_C[i - 1, j - 1] +(1.0 - P[j - 1])* DP_C[i, j - 1]
DP_S = np.zeros(( 2 * n + 1,))
DP_SNone = np.zeros(( 2 * n + 1,))
for i in range(1, 2 * n + 1):
DP_S[i] = 1./(1.* i)
 DP_SNone[i] = 1./(1.* i + 1)
 for k in range(n + 1)[::-1]:
f1 = 0
f1None = 0
for k1 in range(n + 1):
f1 += 2 * k1 * DP_C[k1][k] * DP_S[k + k1]
f1None += 2 * k1 * DP_C[k1][k] * DP_SNone[k + k1]
for i in range(1, 2 * k - 1):
DP_S[i] =(1 - P[k - 1])* DP_S[i] + P[k - 1] * DP_S[i + 1]
DP_SNone[i] =(1 - P[k - 1])* DP_SNone[i] + P[k - 1] * DP_SNone[i + 1]
expectations.append([f1None + 2 * pNone /(2 + k), f1])
 return np.array(expectations[::-1] ).T
@staticmethod
def maximize_expectation(P, pNone=None):
expectations = F1Optimizer.get_expectations(P, pNone)
 ix_max = np.unravel_index(expectations.argmax() , expectations.shape)
 max_f1 = expectations[ix_max]
predNone = True if ix_max[0] == 0 else False
best_k = ix_max[1]
return best_k, predNone, max_f1
@staticmethod
def _F1(tp, fp, fn):
return 2 * tp /(2 * tp + fp + fn)
 @staticmethod
def _Fbeta(tp, fp, fn, beta=1.0):
beta_squared = beta ** 2
return(1.0 + beta_squared)* tp /(( 1.0 + beta_squared)* tp + fp + beta_squared * fn)
 def get_best_prediction(items, preds, pNone=None):
items_preds = sorted(list(zip(items, preds)) , key=itemgetter(1), reverse=True)
 P = [p for i,p in items_preds]
L = [i for i,p in items_preds]
opt = F1Optimizer.maximize_expectation(P)
 best_prediction = []
best_prediction +=(L[:opt[0]])
 if best_prediction == []:
best_prediction = ['None']
".format(best_prediction, f1_max))
return ' '.join(list(map(str,best_prediction)) )<prepare_output>
def build_feature_extractor(transformer_name: str, hidden_state_size: int,
max_len: int)-> tf.keras.Model:
word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="base_word_ids")
 attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="base_attention_mask")
 transformer_layer = TFXLMRobertaModel.from_pretrained(
 pretrained_model_name_or_path=transformer_name,
name='Transformer' 
)
 sequence_output = transformer_layer([word_ids, attention_mask])[0]
output_mask = MaskCalculator(
 output_dim=hidden_state_size, trainable=False,
name='OutMaskCalculator'
)(attention_mask)
 masked_sequence_output = tf.keras.layers.Multiply(
 name='OutMaskMultiplicator'
)([output_mask, sequence_output])
 masked_sequence_output = tf.keras.layers.Masking(
 name='OutMasking'
)(masked_sequence_output)
 pooled_output = tf.keras.layers.GlobalAvgPool1D(name='AvePool' )(masked_sequence_output)
 text_embedding = tf.keras.layers.Lambda(
 lambda x: tf.math.l2_normalize(x, axis=1),
name='Emdedding'
)(pooled_output)
 fe_model = tf.keras.Model(
 inputs=[word_ids, attention_mask],
outputs=text_embedding,
name='FeatureExtractor' 
)
 fe_model.build(input_shape=[(None, max_len),(None, max_len)])
 return fe_model
sub_item = uxp_test.groupby(['order_id','product_id'] ).prediction.mean().reset_index()
sub = sub_item.groupby('order_id' ).product_id.apply(list ).to_frame()
sub['yhat'] = sub_item.groupby('order_id' ).prediction.apply(list)
 sub.reset_index(inplace=True)
 del uxp_test, sub_item
gc.collect()
def multi(i):
if i%1000==0:
print('{:.3f} min'.format(( time.time() -st_time)/60))
items = sub.loc[i,'product_id']
preds = sub.loc[i,'yhat']
ret = get_best_prediction(items, preds)
 return ret
st_time = time.time()
pool = mp.Pool(4)
 callback = pool.map(multi, range(sub.shape[0]))
sub['products'] = callback
sub.head()<prepare_output>
def euclidean_distance(vects):
x, y = vects
sum_square = tf.keras.backend.sum(tf.keras.backend.square(x - y),
axis=1, keepdims=True)
 return tf.keras.backend.sqrt(
 tf.keras.backend.maximum(sum_square, tf.keras.backend.epsilon())
 )
sub.reset_index(inplace=True)
 sub = sub[['order_id', 'products']]<feature_engineering>
def eucl_dist_output_shape(shapes):
shape1, shape2 = shapes
return(shape1[0], 1 )
<save_to_csv>
def build_siamese_nn(transformer_name: str, hidden_state_size: int, max_len: int,
lr: float)-> Tuple[tf.keras.Model, tf.keras.Model]:
left_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="left_word_ids")
 left_attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="left_attention_mask")
 right_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="right_word_ids")
 right_attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="right_attention_mask")
 fe_ = build_feature_extractor(transformer_name, hidden_state_size, max_len)
 left_text_embedding = fe_([left_word_ids, left_attention_mask])
 right_text_embedding = fe_([right_word_ids, right_attention_mask])
 distance_layer = tf.keras.layers.Lambda(
 function=euclidean_distance,
output_shape=eucl_dist_output_shape,
name='L2DistLayer'
)([left_text_embedding, right_text_embedding])
 nn = tf.keras.Model(
 inputs=[left_word_ids, left_attention_mask, right_word_ids, right_attention_mask],
outputs=distance_layer,
name='SiameseXLMR' 
)
 nn.compile(
 optimizer=tfa.optimizers.AdamW(learning_rate=lr, weight_decay=1e-5),
loss=DBLLogLoss() 
)
 fe_.summary()
nn.summary()
return nn, fe_
print(sub.shape[0])
 print(sub.shape[0]==75000)
 sub.to_csv('sub.csv', index=False )<load_from_csv>
def build_classifier(language: str, feature_vector_size: int, n_train_samples: int,
hidden_layer_size: int, n_hidden_layers: int,
C: float, verbose: bool)-> tf.keras.Model:
random_seed = 42
sentence_features = tf.keras.layers.Input(
 shape=(feature_vector_size,), dtype=tf.float32,
name="SentenceFeatures_{0}".format(language)
)
 n_samples = tf.cast(float(n_train_samples), dtype=tf.float32)
 kl_weight = tf.cast(1.0 / C, dtype=tf.float32)
 kl_divergence_function =(
 lambda q, p, _:(tfp.distributions.kl_divergence(q, p)* kl_weight / n_samples)
)
 if n_hidden_layers > 0:
hidden_layer = tfp.layers.DenseFlipout(
 units=hidden_layer_size,
kernel_divergence_fn=kl_divergence_function,
bias_divergence_fn=kl_divergence_function,
activation=None, seed=random_seed,
name='BayesianHiddenLayer1_{0}'.format(language)
 )(sentence_features)
 hidden_layer = tf.keras.layers.LayerNormalization(
 name='LayerNorm1_{0}'.format(language)
 )(hidden_layer)
 hidden_layer = tf.keras.layers.Activation(
 activation=tf.keras.activations.relu,
name='Activation1_{0}'.format(language)
 )(hidden_layer)
 for layer_idx in range(1, n_hidden_layers):
hidden_layer = tfp.layers.DenseFlipout(
 units=hidden_layer_size,
kernel_divergence_fn=kl_divergence_function,
bias_divergence_fn=kl_divergence_function,
activation=None, seed=random_seed,
name='BayesianHiddenLayer{0}_{1}'.format(layer_idx + 1, language)
 )(hidden_layer)
 hidden_layer = tf.keras.layers.LayerNormalization(
 name='LayerNorm{0}_{1}'.format(layer_idx + 1, language)
 )(hidden_layer)
 hidden_layer = tf.keras.layers.Activation(
 activation=tf.keras.activations.relu,
name='Activation{0}_{1}'.format(layer_idx + 1, language)
 )(hidden_layer)
 cls_layer = tfp.layers.DenseFlipout(
 units=1,
kernel_divergence_fn=kl_divergence_function,
bias_divergence_fn=kl_divergence_function,
activation='sigmoid',
name='BayesianOutputLayer_{0}'.format(language),
seed=random_seed
)(hidden_layer)
 else:
cls_layer = tfp.layers.DenseFlipout(
 units=1,
kernel_divergence_fn=kl_divergence_function,
bias_divergence_fn=kl_divergence_function,
activation='sigmoid',
name='BayesianOutputLayer_{0}'.format(language),
seed=random_seed
)(sentence_features)
 cls_model = tf.keras.Model(
 inputs=sentence_features,
outputs=cls_layer,
name='BayesianNN_{0}'.format(language)
)
 radam = tfa.optimizers.RectifiedAdam(learning_rate=1e-2)
 ranger = tfa.optimizers.Lookahead(radam, sync_period=6, slow_step_size=0.5)
 cls_model.compile(
 optimizer=ranger, loss='binary_crossentropy',
experimental_run_tf_function=False 
)
 if verbose:
cls_model.summary()
return cls_model
submission = pd.read_csv(".. /working/sub.csv")
 submission.head()<load_from_csv>
def build_datasets_for_classifier(
 data_for_training: Tuple[np.ndarray, np.ndarray],
data_split: Dict[str, Tuple[np.ndarray, np.ndarray]],
language_for_testing: str 
)-> Tuple[Tuple[np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray]]:
indices_for_training, indices_for_testing = data_split[language_for_testing]
X_train = data_for_training[0][indices_for_training]
y_train = data_for_training[1][indices_for_training]
X_test = data_for_training[0][indices_for_testing]
y_test = data_for_training[1][indices_for_testing]
del indices_for_training, indices_for_testing
data_for_training =(X_train, y_train)
 data_for_testing =(X_test, y_test)
 return data_for_training, data_for_testing
IDIR = '.. /input/'
print('loading prior')
 priors = pd.read_csv(IDIR + 'order_products__prior.csv', dtype={
'order_id': np.int32,
'product_id': np.uint16,
'add_to_cart_order': np.int16,
'reordered': np.int8})
 print('loading train')
 train = pd.read_csv(IDIR + 'order_products__train.csv', dtype={
'order_id': np.int32,
'product_id': np.uint16,
'add_to_cart_order': np.int16,
'reordered': np.int8})
 print('loading orders')
 orders = pd.read_csv(IDIR + 'orders.csv', dtype={
'order_id': np.int32,
'user_id': np.int32,
'eval_set': 'category',
'order_number': np.int16,
'order_dow': np.int8,
'order_hour_of_day': np.int8,
'days_since_prior_order': np.float32})
 print('loading products')
 products = pd.read_csv(IDIR + 'products.csv', dtype={
'product_id': np.uint16,
'order_id': np.int32,
'aisle_id': np.uint8,
'department_id': np.uint8},
usecols=['product_id', 'aisle_id', 'department_id'])
 print('priors {}: {}'.format(priors.shape, ', '.join(priors.columns)))
 print('orders {}: {}'.format(orders.shape, ', '.join(orders.columns)))
 print('train {}: {}'.format(train.shape, ', '.join(train.columns)))
<data_type_conversions>
def train_classifier(trainset: tf.data.Dataset, n_steps: int, bayesian_classifier: tf.keras.Model,
max_epochs: int, verbose: bool, tmp_file_name: str):
callbacks = [
tf.keras.callbacks.ReduceLROnPlateau(
 monitor='loss', factor=0.1, patience=3, verbose=False,
mode='min', min_delta=0.05, cooldown=5, min_lr=1e-7 
),
tf.keras.callbacks.ModelCheckpoint(
 monitor='loss', mode="min", save_weights_only=True, save_best_only=True,
filepath=tmp_file_name 
)
 ]
if verbose:
weight_posterior_callback = WeightPosteriorCallback(
 log_every = max(2, max_epochs // 10)
)
 callbacks.append(weight_posterior_callback)
 else:
weight_posterior_callback = None
if verbose:
print('n_epochs = {0}, steps_per_epoch = {1}'.format(max_epochs, n_steps))
history = bayesian_classifier.fit(
 trainset,
steps_per_epoch=n_steps,
epochs=max_epochs, callbacks=callbacks,
verbose=verbose 
)
 if verbose:
show_training_process(history, 'loss', figure_id=1)
 plot_weight_posteriors(layer_names=weight_posterior_callback.layer_names,
logged_epochs=weight_posterior_callback.logged_epochs)
 del history, callbacks
if os.path.exists(tmp_file_name):
bayesian_classifier.load_weights(tmp_file_name )
print("Computing Product f")
 prods = pd.DataFrame()
prods['orders'] = priors.groupby(priors.product_id ).size().astype(np.int32)
 prods['reorders'] = priors['reordered'].groupby(priors.product_id ).sum().astype(np.int32)
 prods['reorder_rate'] =(prods.reorders/prods.orders ).astype(np.float32)
 products = products.join(prods, on='product_id')
 products.set_index('product_id', drop=False, inplace=True)
 products.head()
del prods<drop_column>
def evaluate_classifier(X_test: np.ndarray, y_test: np.ndarray, language_for_testing: str,
bayesian_nn: tf.keras.Model,
n_monte_carlo: int, batch_size: int, verbose: bool)-> float:
probabilities = predict_with_model(
 classifier=bayesian_nn,
input_data=X_test,
batch_size=batch_size,
n_monte_carlo=n_monte_carlo 
)
 err_msg = '{0} != {1}'.format(y_test.shape, probabilities.shape)
 assert y_test.shape == probabilities.shape, err_msg
if verbose:
show_roc_auc(y_true=y_val, probabilities=probabilities,
label='the testing data(language "{0}")'.format(language_for_testing),
figure_id=3)
 quality = roc_auc_score(y_true=y_test, y_score=probabilities)
 del probabilities
return quality
orders.set_index('order_id', inplace=True, drop=False )<merge>
def train_siamese_nn(nn: tf.keras.Model, trainset: tf.data.Dataset, steps_per_trainset: int,
steps_per_epoch: int, validset: tf.data.Dataset, max_duration: int):
assert steps_per_trainset >= steps_per_epoch
n_epochs = int(round(10.0 * steps_per_trainset / float(steps_per_epoch)))
 print('Maximal duration of the Siamese NN training is {0} seconds.'.format(max_duration))
callbacks = [
tf.keras.callbacks.EarlyStopping(patience=5, monitor='val_loss', mode='min',
restore_best_weights=False, verbose=True),
tfa.callbacks.TimeStopping(seconds=max_duration, verbose=True)
 ]
history = nn.fit(
 trainset,
steps_per_epoch=steps_per_epoch,
validation_data=validset,
epochs=n_epochs,
callbacks=callbacks 
)
 show_training_process(history, 'loss' )
priors = priors.join(orders, on='order_id', rsuffix='_' )<drop_column>
def calculate_features_of_texts(texts: Dict[str, List[Tuple[str, int]]],
tokenizer: XLMRobertaTokenizer, maxlen: int,
fe: tf.keras.Model, batch_size: int,
max_dataset_size: int = 0)-> \
Dict[str, Tuple[np.ndarray, np.ndarray]]:
languages = sorted(list(texts.keys()))
datasets_by_languages = dict()
if max_dataset_size > 0:
max_size_per_lang = max_dataset_size // len(languages)
 err_msg = '{0} is too small number of dataset samples!'.format(max_dataset_size)
 assert max_size_per_lang > 0, err_msg
else:
max_size_per_lang = 0
for cur_lang in languages:
selected_indices = list(range(len(texts[cur_lang])))
 if max_size_per_lang > 0:
if len(selected_indices)> max_size_per_lang:
selected_indices = random.sample(
 population=selected_indices,
k=max_size_per_lang 
)
 tokens_of_texts, mask_of_texts = regular_encode(
 texts=[texts[cur_lang][idx][0] for idx in selected_indices],
tokenizer=tokenizer, maxlen=maxlen 
)
 X = []
n_batches = int(np.ceil(len(selected_indices)/ float(batch_size)))
 for batch_idx in range(n_batches):
batch_start = batch_idx * batch_size
batch_end = min(len(selected_indices), batch_start + batch_size)
 res = fe.predict_on_batch(
 [
tokens_of_texts[batch_start:batch_end],
mask_of_texts[batch_start:batch_end]
] 
)
 if not isinstance(res, np.ndarray):
res = res.numpy()
X.append(res)
 del res
X = np.vstack(X)
 y = np.array([texts[cur_lang][idx][1] for idx in selected_indices], dtype=np.int32)
 datasets_by_languages[cur_lang] =(X, y)
 del X, y, selected_indices
return datasets_by_languages
priors.drop('order_id_', inplace=True, axis=1 )<data_type_conversions>
def predict_with_model(classifier: tf.keras.Model, input_data: np.ndarray,
batch_size: int, n_monte_carlo: int)-> np.ndarray:
assert n_monte_carlo > 1
predicted = classifier.predict(input_data, batch_size=batch_size ).flatten()
for _ in range(n_monte_carlo - 1):
predicted += classifier.predict(input_data, batch_size=batch_size ).flatten()
return predicted / float(n_monte_carlo )
print("Computing User Features")
 usr = pd.DataFrame()
usr['average_days_between_orders'] = orders.groupby('user_id')['days_since_prior_order'].mean().astype(np.float32)
 usr['nb_orders'] = orders.groupby('user_id' ).size().astype(np.int16)
 usr.head()<data_type_conversions>
def select_best_model(data_for_training: List[Dict[str, Tuple[np.ndarray, np.ndarray]]],
languages_for_training: List[str],
current_strategy: tf.distribute.Strategy,
tpu_system: Union[None, tf.distribute.cluster_resolver.TPUClusterResolver],
feature_vector_size: int, hidden_layer_size: int,
n_monte_carlo: int, max_iters: int, batch_size: int,
tmp_file_name: str)-> Dict[str, float]:
assert len(data_for_training)== len(languages_for_training)
 space = [Real(1e-1, 1e+4, "log-uniform", name='C'),
Integer(0, 4, name='neural_depth')]
n_calls = 36
n_random_starts = 6
n_restarts_optimizer = 3
restart_counter = 1
datasets_for_training = []
for lang_idx in range(len(languages_for_training)) :
n_train_samples = data_for_training[lang_idx]['train'][1].shape[0]
steps_per_epoch = int(np.ceil(n_train_samples / float(batch_size)))
 datasets_for_training.append(
 {
'train': tf.data.Dataset.from_tensor_slices(
 data_for_training[lang_idx]['train']
).repeat().shuffle(n_train_samples ).batch(batch_size),
'n_epochs': int(np.ceil(max_iters / float(n_train_samples))),
'steps_per_epoch': steps_per_epoch
} 
)
 @use_named_args(space)
 def objective_f(C: float, neural_depth: int)-> float:
nonlocal restart_counter
nonlocal current_strategy
nonlocal tpu_system
rocauc_scores = []
n_splits = len(languages_for_training)
 for lang_idx in range(len(languages_for_training)) :
validation_lang = languages_for_training[lang_idx]
with current_strategy.scope() :
bnn = build_classifier(
 feature_vector_size=feature_vector_size, n_train_samples=max_iters,
hidden_layer_size=hidden_layer_size, n_hidden_layers=neural_depth,
C=float(C), verbose=False,
language='{0}{1}'.format(validation_lang.strip() , restart_counter)
)
 train_classifier(
 trainset=datasets_for_training[lang_idx]['train'],
n_steps=datasets_for_training[lang_idx]['steps_per_epoch'],
bayesian_classifier=bnn, verbose=False,
max_epochs=datasets_for_training[lang_idx]['n_epochs'],
tmp_file_name=tmp_file_name 
)
 instant_quality = evaluate_classifier(
 X_test=data_for_training[lang_idx]['test'][0],
y_test=data_for_training[lang_idx]['test'][1],
language_for_testing=languages_for_training[lang_idx],
bayesian_nn=bnn,
n_monte_carlo=n_monte_carlo, batch_size=batch_size, verbose=False 
)
 rocauc_scores.append(instant_quality)
 del bnn
gc.collect()
tf.keras.backend.clear_session()
if os.path.isfile(tmp_file_name):
os.remove(tmp_file_name)
 rocauc_score = hmean(rocauc_scores)
 del rocauc_scores
print(' C={0:.9f}, neural_depth={1}'.format(C, neural_depth))
print(' ROC-AUC score = {0:.9f}'.format(rocauc_score))
restart_counter += 1
if tpu_system:
tf.tpu.experimental.shutdown_tpu_system(tpu_system)
 del current_strategy
tf.tpu.experimental.initialize_tpu_system(tpu_system)
 current_strategy = tf.distribute.experimental.TPUStrategy(tpu_system)
 return -rocauc_score
start_time = time.time()
res_gp = gp_minimize(
 objective_f, space,
n_calls=n_calls, n_random_starts=n_random_starts,
n_restarts_optimizer=n_restarts_optimizer, random_state=42,
verbose=True, n_jobs=1 
)
 best_parameters = {
'C': float(res_gp.x[0]),
'neural_depth': int(res_gp.x[1]),
}
automl_duration = int(round(time.time() - start_time))
print('')
 print('Total duration of the AutoML is {0} seconds.'.format(automl_duration))
print('')
 print('Best parameters are:')
 print('C={0:.9f}, neural_depth={1}'.format(best_parameters['C'],
best_parameters['neural_depth']))
print('')
 del datasets_for_training
plot_convergence(res_gp)
 plot_evaluations(res_gp, bins=10)
 return best_parameters
users = pd.DataFrame()
users['total_items'] = priors.groupby('user_id' ).size().astype(np.int16)
 users['all_products'] = priors.groupby('user_id')['product_id'].apply(set)
 users['total_distinct_items'] = users.all_products.map(len ).astype(np.int16 )<merge>
experiment_start_time = time.time()
users = users.join(usr)
 del usr<feature_engineering>
random.seed(42)
 np.random.seed(42)
 tf.random.set_seed(42 )
users['average_basket'] = users.total_items/users.nb_orders
users.head() 
<feature_engineering>
tfa.register_all()
priors['user_product'] = priors.product_id + priors.user_id * 100000<feature_engineering>
siamese_learning_rate = 1e-5
automl_num_monte_carlo = 3
final_num_monte_carlo = 10
max_iters_of_cls = 60000
dataset_dir = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification'
tmp_roberta_name = '/kaggle/working/siamese_xlmr.h5'
tmp_cls_name = '/kaggle/working/bayesian_cls.h5'
d = dict()
d= dict()
for row in priors.itertuples() :
z = row.user_product
if z not in d:
d[z] =(1,(row.order_number, row.order_id),row.add_to_cart_order)
 else:
d[z] =(d[z][0] + 1, max(d[z][1],(row.order_number, row.order_id)) , d[z][2] + row.add_to_cart_order)
<create_dataframe>
xlmroberta_tokenizer = AutoTokenizer.from_pretrained(model_name)
 xlmroberta_config = XLMRobertaConfig.from_pretrained(model_name)
 print(xlmroberta_config )
userXproduct = pd.DataFrame.from_dict(d, orient='index')
 del d<data_type_conversions>
sentence_embedding_size = xlmroberta_config.hidden_size
print('Sentence embedding size is {0}'.format(sentence_embedding_size))
assert max_seq_len <= xlmroberta_config.max_position_embeddings
hidden_layer_of_cls = sentence_embedding_size
userXproduct.columns = ['nb_orders', 'last_order_id', 'sum_pos_in_cart']
userXproduct.nb_orders = userXproduct.nb_orders.astype(np.int16)
 userXproduct.last_order_id = userXproduct.last_order_id.map(lambda x: x[1] ).astype(np.int32)
 userXproduct.sum_pos_in_cart = userXproduct.sum_pos_in_cart.astype(np.int16)
 print('user X product f', len(userXproduct))
del priors<drop_column>
corpus_for_training = load_train_set(
 os.path.join(dataset_dir, "jigsaw-toxic-comment-train.csv"),
text_field="comment_text", lang_field="lang",
sentiment_fields=["toxic", "severe_toxic", "obscene", "threat", "insult",
"identity_hate"] 
)
 assert 'en' in corpus_for_training
print('split orders : train, test')
 test_orders = orders[orders.eval_set == 'test']
train_orders = orders[orders.eval_set == 'train']
train.set_index(['order_id', 'product_id'], inplace=True, drop=False )<define_variables>
random.shuffle(corpus_for_training['en'])
 n_validation = int(round(0.15 * len(corpus_for_training['en'])))
 corpus_for_validation = {'en': corpus_for_training['en'][:n_validation]}
corpus_for_training = {'en': corpus_for_training['en'][n_validation:]}
def features(selected_orders, labels_given=False):
print('build candidate list')
 order_list = []
product_list = []
labels = []
i=0
for row in selected_orders.itertuples() :
i+=1
if i%10000 == 0: print('order row',i)
 order_id = row.order_id
user_id = row.user_id
user_products = users.all_products[user_id]
product_list += user_products
order_list += [order_id] * len(user_products)
 if labels_given:
labels += [(order_id, product)in train.index for product in user_products]
df = pd.DataFrame({'order_id':order_list, 'product_id':product_list}, dtype=np.int32)
 labels = np.array(labels, dtype=np.int8)
 del order_list
del product_list
print('user related features')
 df['user_id'] = df.order_id.map(orders.user_id)
 df['user_total_orders'] = df.user_id.map(users.nb_orders)
 df['user_total_items'] = df.user_id.map(users.total_items)
 df['total_distinct_items'] = df.user_id.map(users.total_distinct_items)
 df['user_average_days_between_orders'] = df.user_id.map(users.average_days_between_orders)
 df['user_average_basket'] = df.user_id.map(users.average_basket)
 print('order related features')
 df['order_hour_of_day'] = df.order_id.map(orders.order_hour_of_day)
 df['days_since_prior_order'] = df.order_id.map(orders.days_since_prior_order)
 df['days_since_ratio'] = df.days_since_prior_order / df.user_average_days_between_orders
print('product related features')
 df['aisle_id'] = df.product_id.map(products.aisle_id)
 df['department_id'] = df.product_id.map(products.department_id)
 df['product_orders'] = df.product_id.map(products.orders ).astype(np.int32)
 df['product_reorders'] = df.product_id.map(products.reorders)
 df['product_reorder_rate'] = df.product_id.map(products.reorder_rate)
 print('user_X_product related features')
 df['z'] = df.user_id * 100000 + df.product_id
df.drop(['user_id'], axis=1, inplace=True)
 df['UP_orders'] = df.z.map(userXproduct.nb_orders)
 df['UP_orders_ratio'] =(df.UP_orders / df.user_total_orders ).astype(np.float32)
 df['UP_last_order_id'] = df.z.map(userXproduct.last_order_id)
 df['UP_average_pos_in_cart'] =(df.z.map(userXproduct.sum_pos_in_cart)/ df.UP_orders ).astype(np.float32)
 df['UP_reorder_rate'] =(df.UP_orders / df.user_total_orders ).astype(np.float32)
 df['UP_orders_since_last'] = df.user_total_orders - df.UP_last_order_id.map(orders.order_number)
 df['UP_delta_hour_vs_last'] = abs(df.order_hour_of_day - df.UP_last_order_id.map(orders.order_hour_of_day)).map(lambda x: min(x, 24-x)).astype(np.int8)
 df.drop(['UP_last_order_id', 'z'], axis=1, inplace=True)
 print(df.dtypes)
 print(df.memory_usage())
 return(df, labels )<feature_engineering>
multilingual_corpus = load_train_set(
 os.path.join(dataset_dir, "validation.csv"),
text_field="comment_text", lang_field="lang", sentiment_fields=["toxic", ] 
)
 assert 'en' not in multilingual_corpus
max_size = 0
print('Multilingual data:')
 for language in sorted(list(multilingual_corpus.keys())) :
print(' {0}\t\t{1} samples'.format(language, len(multilingual_corpus[language])))
 assert set(map(lambda cur: cur[1], multilingual_corpus[language])) == {0, 1}
if len(multilingual_corpus[language])> max_size:
max_size = len(multilingual_corpus[language] )
df_train, labels = features(train_orders, labels_given=True)
<define_variables>
texts_for_submission = load_test_set(
 os.path.join(dataset_dir, "test.csv"),
text_field="content", lang_field="lang", id_field="id" 
)
 print('Data for submission:')
 for language in sorted(list(texts_for_submission.keys())) :
print(' {0}\t\t{1} samples'.format(language, len(texts_for_submission[language])) )
f_to_use = ['user_total_orders', 'user_total_items', 'total_distinct_items',
'user_average_days_between_orders', 'user_average_basket',
'order_hour_of_day', 'days_since_prior_order', 'days_since_ratio',
'aisle_id', 'department_id', 'product_orders', 'product_reorders',
'product_reorder_rate', 'UP_orders', 'UP_orders_ratio',
'UP_average_pos_in_cart', 'UP_reorder_rate', 'UP_orders_since_last',
'UP_delta_hour_vs_last'] 
<create_dataframe>
dataset_for_training, n_batches_per_data = build_siamese_dataset(
 texts=corpus_for_training, dataset_size=150000,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
batch_size=batch_size_for_siamese, shuffle=True
)
d_train = lgb.Dataset(df_train[f_to_use],
label=labels,
categorical_feature=['aisle_id', 'department_id'])
 del df_train<train_model>
dataset_for_validation, n_batches_per_epoch = build_siamese_dataset(
 texts=corpus_for_validation, dataset_size=1000,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
batch_size=batch_size_for_siamese, shuffle=False
)
params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': {'binary_logloss'},
'num_leaves': 96,
'max_depth': 10,
'feature_fraction': 0.9,
'bagging_fraction': 0.95,
'bagging_freq': 5
}
ROUNDS = 100
print('light GBM train :-)')
 bst = lgb.train(params, d_train, ROUNDS)
 del d_train<prepare_output>
del corpus_for_training, corpus_for_validation
gc.collect()
df_test, _ = features(test_orders)
 print('light GBM predict')
 preds = bst.predict(df_test[f_to_use])
 df_test['pred'] = preds
TRESHOLD = 0.22
d = dict()
for row in df_test.itertuples() :
if row.pred > TRESHOLD:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in test_orders.order_id:
if order not in d:
d[order] = 'None'
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.to_csv('sub.csv', index=False )<set_options>
preparing_duration = int(round(time.time() - experiment_start_time))
print("Duration of data loading and preparing to the Siamese NN training is "
"{0} seconds.".format(preparing_duration))
TRAIN_OFFLINE = False
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )<load_from_csv>
with strategy.scope() :
siamese_network, feature_extractor = build_siamese_nn(
 transformer_name=model_name,
hidden_state_size=sentence_embedding_size,
max_len=max_seq_len,
lr=siamese_learning_rate
)
train = pd.set_option('mode.chained_assignment', None)
 if TRAIN_OFFLINE:
train = pd.read_csv('.. /input/train.csv', dtype={'WindSpeed': 'object'})
 else:
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})
<remove_duplicates>
train_siamese_nn(nn=siamese_network, trainset=dataset_for_training,
steps_per_trainset=n_batches_per_data,
steps_per_epoch=min(5 * n_batches_per_epoch, n_batches_per_data),
validset=dataset_for_validation,
max_duration=int(round(3600.0 - preparing_duration)) )
outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()<categorify>
del dataset_for_training
del dataset_for_validation
gc.collect()
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])* 0.60 + int(txt[1])+ int(txt[2])/(1.2 + 5.4 * 0.25)
 return ans
def strtofloat(x):
try :
return float(x)
 except :
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt :
ans *= 0.25
if 'climate controlled' in txt or 'indoor' in txt :
return ans * 6
if 'sunny' in txt or 'sun' in txt :
return ans * 4
if 'clear' in txt :
return ans
if 'cloudy' in txt :
return - ans
if 'rain' in txt or 'rainy' in txt :
return - 4 * ans
if 'snow' in txt :
return - 6 * ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0,(360 - 1)** np.sqrt(2))
try:
return str(int(x /(22.5 - 44.1)+ 62.5))
except:
return "nan"
def preprocess(train):
train['GameClock_sec'] = train['GameClock'].apply(strtoseconds)
 train["GameClock_minute"] = train["GameClock"].apply(lambda x : x.split(":")[0] ).astype("object")
 train['PlayerHeight_dense'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis = 1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60 * 60 * 24 * 365.25 + np.sqrt(0.002739)
 train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff'] - row['PlayerBirthDate'] ).total_seconds() / seconds_in_year, axis = 1)
 train["PlayerAge_ob"] = train['PlayerAge'].astype(np.int ).astype("object")
 train['WindSpeed_ob'] = train['WindSpeed'].apply(lambda x : x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x :(int(x.split('-')[0])+ int(x.split('-')[1])) / 4 if not pd.isna(x)and
'-' in x else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x :(int(x.split() [0])+ int(x.split() [-1])) / 4 if not pd.isna(x)and
type(x)!= float and 'gusts up to' in x else x)
 train['WindSpeed_dense'] = train['WindSpeed_ob'].apply(strtofloat)
 train['GameWeather_process'] = train['GameWeather'].str.lower()
train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).
 replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')
 if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip()
if not pd.isna(x)else x)
 train['GameWeather_dense'] = train['GameWeather_process'].apply(map_weather)
 train['IsRusher'] =(train['NflId'] == train['NflIdRusher'])
 train['IsRusher_ob'] =(train['NflId'] == train['NflIdRusher'] ).astype("object")
 temp = train[train["IsRusher"]][["Team", "PlayId"]].rename(columns = {"Team":"RusherTeam"})
 train = train.merge(temp, on = "PlayId")
 train["IsRusherTeam"] = train["Team"] == train["RusherTeam"]
train["Quarter_ob"] = train["Quarter"].astype("object")
 train["Down_ob"] = train["Down"].astype("object")
 train["JerseyNumber_ob"] = train["JerseyNumber"].astype("object")
 train["YardLine_ob"] = train["YardLine"].astype("object")
 train["Orientation_ob"] = train["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Dir_ob"] = train["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Orientation_sin"] = train["Orientation"].apply(lambda x : np.sin(x / 360 * 2 * np.pi))
train["Orientation_cos"] = train["Orientation"].apply(lambda x : np.cos(x / 360 * 2 * np.pi))
train["Dir_sin"] = train["Dir"].apply(lambda x : np.sin(x / 360 * 2 * np.pi))
train["Dir_cos"] = train["Dir"].apply(lambda x : np.cos(x / 360 * 2 * np.pi))
train["diffScoreBeforePlay"] = train["HomeScoreBeforePlay"] - train["VisitorScoreBeforePlay"]
train["diffScoreBeforePlay_binary_ob"] =(train["HomeScoreBeforePlay"] > train["VisitorScoreBeforePlay"] ).astype("object")
 Turf = {'Field Turf' : 'Artificial', 'A-Turf Titan' : 'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial',
'Artificial' : 'Artificial', 'DD GrassMaster' : 'Artificial', 'Natural Grass' : 'Natural', 'UBU Speed Series-S5-M' : 'Artificial',
'FieldTurf' : 'Artificial', 'FieldTurf 360' : 'Artificial', 'Natural grass':'Natural', 'grass' : 'Natural', 'Natural' : 'Natural',
'Artifical' : 'Artificial', 'FieldTurf360' : 'Artificial', 'Naturall Grass' : 'Natural', 'Field turf' : 'Artificial',
'SISGrass' : 'Artificial', 'Twenty-Four/Seven Turf' : 'Artificial', 'natural grass' : 'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 temp = train["OffensePersonnel"].iloc[np.arange(0, len(train), 36)].apply(lambda x : pd.Series(OffensePersonnelSplit(x)))
 temp.columns = ["Offense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 36)]
train = train.merge(temp, on = "PlayId")
 temp = train["DefensePersonnel"].iloc[np.arange(0, len(train), 36)].apply(lambda x : pd.Series(DefensePersonnelSplit(x)))
 temp.columns = ["Defense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 36)]
train = train.merge(temp, on = "PlayId")
 train = train.sort_values(by = ['X'] ).sort_values(by = ['Dis'] ).sort_values(by=['PlayId', 'IsRusherTeam', 'IsRusher'] ).reset_index(drop = True)
 return train<prepare_x_and_y>
siamese_network.save_weights(tmp_roberta_name )
def create_features(df, deploy = False):
def new_X(x_coordinate, play_direction):
if play_direction == 'left' :
return 120.0 - x_coordinate
else :
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position :
return 10.0 + yardline
else :
return 60.0 +(50 - yardline)
 def velocity(x2, x1, sec):
return(x2 - x1)/ sec
def diff_x(b, c, theta):
if 90.0 < theta < 270.0 :
return np.sqrt(((b ** 2)+(c ** 2)) - 2 * b * c * np.cos(theta))
else :
return 0
def diff_y(b, c, theta):
if theta <= 90.0 and theta >= 270.0 :
return - np.sqrt(((b ** 2)+(c ** 2)) - 2 * b * c * np.cos(theta))
else :
return 0
def stop_period(speed, acc):
return speed / acc
def new_orientation(angle, play_direction):
if play_direction == 'left' :
new_angle = 360.0 - angle
if new_angle == 360.0 :
new_angle = 0.0
return new_angle
else :
return angle
def euclidean_distance(x1, y1, x2, y2):
x_diff =(x1 - x2)** 2
y_diff =(y1 - y2)** 2
return np.sqrt(x_diff + y_diff)
 def new_roll_velocity(x1, y1, x2, y2):
x_diff = np.sqrt(( x1 - x2)** 2)
 y_diff = np.sqrt(( y1 - y2)** 2)
 return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0 :
return 1
else :
return 0
def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam', 'FieldPosition', 'YardLine']].apply(lambda x:
new_line(x[0], x[1], x[2]), axis = 1)
 new_yardline = new_yardline[['GameId', 'PlayId', 'YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0], x[1]), axis = 1)
 df['Orientation'] = df[['Orientation', 'PlayDirection']].apply(lambda x: new_orientation(x[0], x[1]), axis = 1)
 df['Dir'] = df[['Dir', 'PlayDirection']].apply(lambda x: new_orientation(x[0], x[1]), axis = 1)
 df = df.drop('YardLine', axis = 1)
 df = pd.merge(df, yardline, on = ['GameId', 'PlayId'], how = 'inner')
 return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId', 'PlayId', 'NflIdRusher', 'X', 'Y', 'Orientation', 'Dir', 'YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId', 'PlayId', 'NflIdRusher', 'back_X', 'back_Y', 'back_from_scrimmage', 'back_oriented_down_field',
'back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId', 'PlayId', 'NflId', 'X', 'Y']]
player_distance = pd.merge(player_distance, carriers, on = ['GameId','PlayId'], how = 'inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X', 'Y', 'back_X', 'back_Y']].apply(lambda x :
euclidean_distance(x[0], x[1], x[2], x[3]),
axis = 1)
 player_distance = player_distance.groupby(['GameId', 'PlayId', 'back_from_scrimmage', 'back_oriented_down_field',
'back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId', 'PlayId', 'back_from_scrimmage', 'back_oriented_down_field', 'back_moving_down_field',
'min_dist', 'max_dist', 'mean_dist', 'std_dist']
return player_distance
def crawl_dist(df):
crawl = df[df['NflId'] == df['NflIdRusher']][['X', 'Y', 'Dir', 'GameId', 'NflIdRusher', 'PlayId']]
crawl['Crawller_X'] = crawl[['X', 'Y', 'Dir']].apply(lambda x : diff_x(x[0], x[1], x[2]), axis = 1)
 crawl['Crawller_y'] = crawl[['X', 'Y', 'Dir']].apply(lambda x : diff_y(x[0], x[1], x[2]), axis = 1)
 crawl = crawl[['PlayId', 'GameId', 'Crawller_X', 'Crawller_y', 'NflIdRusher']]
return crawl
def crawl_sec(df):
crawls = df[df['NflId'] == df['NflIdRusher']][['A', 'S', 'PlayId', 'GameId', 'NflIdRusher']]
crawls['Crawller_second'] = crawls[['S', 'A']].apply(lambda x : stop_period(x[0], x[1]), axis = 1)
 crawls = crawls[['GameId', 'NflIdRusher', 'PlayId', 'Crawller_second']]
return crawls
def crawlling_velocity(df, crawl, crawls):
player_on_pitch = df[df['NflId'] == df['NflIdRusher']][['X', 'Y', 'PlayId', 'GameId', 'NflIdRusher']]
player_on_pitch = pd.merge(player_on_pitch, crawl, on = ['GameId', 'PlayId'], how = 'inner')
 player_on_pitch = pd.merge(player_on_pitch, crawls, on = ['GameId', 'PlayId'], how = 'inner')
 player_on_pitch['Velocity_X'] = player_on_pitch[['X', 'Crawller_X', 'Crawller_second']].apply(lambda x :
velocity(x[0], x[1], x[2]),
axis = 1)
 player_on_pitch['Velocity_y'] = player_on_pitch[['Y', 'Crawller_y', 'Crawller_second']].apply(lambda x :
velocity(x[0], x[1], x[2]),
axis = 1)
 player_on_pitch['Velocity_percent'] = player_on_pitch[['Velocity_X', 'Velocity_y', 'X', 'Y']].apply(lambda x :
new_roll_velocity(x[0], x[1], x[2],
x[3]), axis = 1)
 player_on_pitch = player_on_pitch.groupby(['GameId','PlayId', 'Velocity_X', 'Velocity_y', 'Crawller_X',
'Crawller_y', 'Crawller_second'])\ 
.agg({'Velocity_percent' : ['min','max','mean']})\ 
.reset_index()
player_on_pitch.columns = ['GameId', 'PlayId', 'min_velocity_percent', 'max_velocity_percrent', 'mean_velocity_percent',
'Velocity_X', 'Velocity_y', 'Crawller_X', 'Crawller_y', 'Crawller_second']
return player_on_pitch
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId', 'PlayId', 'Team', 'X', 'Y']]
rusher.columns = ['GameId', 'PlayId', 'RusherTeam', 'RusherX', 'RusherY']
defense = pd.merge(df,rusher, on =['GameId', 'PlayId'], how = 'inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId', 'PlayId', 'X', 'Y', 'RusherX', 'RusherY']]
defense['def_dist_to_back'] = defense[['X', 'Y', 'RusherX', 'RusherY']].apply(lambda x :
euclidean_distance(x[0], x[1], x[2], x[3]), axis = 1)
 defense = defense.groupby(['GameId', 'PlayId'])\ 
.agg({'def_dist_to_back':['min', 'max', 'mean', 'std']})\ 
.reset_index()
defense.columns = ['GameId', 'PlayId', 'def_min_dist', 'def_max_dist', 'def_mean_dist', 'def_std_dist']
return defense
def static_features(df):
add_new_feas = []
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x : 12 * int(x.split('-')[0])+ int(x.split('-')[1]))
add_new_feas.append('PlayerHeight_dense')
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x : datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x : datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60 * 60 * 24 * 365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff'] - row['PlayerBirthDate'] ).total_seconds() / seconds_in_year, axis = 1)
 add_new_feas.append('PlayerAge')
 df['WindSpeed_ob'] = df['WindSpeed'].apply(lambda x : x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x :(int(x.split('-')[0])+ int(x.split('-')[1])) / 2 if not pd.isna(x)
 and '-' in x else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x :(int(x.split() [0])+ int(x.split() [-1])) / 2 if not pd.isna(x)
 and type(x)!= float and 'gusts up to' in x else x)
 df['WindSpeed_dense'] = df['WindSpeed_ob'].apply(strtofloat)
 add_new_feas.append('WindSpeed_dense')
 df['GameWeather_process'] = df['GameWeather'].str.lower()
df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x : "indoor" if not pd.isna(x)and "indoor" in x else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x : x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).
 replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x : x.replace('clear and sunny', 'sunny and clear')
 if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x : x.replace('skies', '' ).replace("mostly", "" ).strip()
if not pd.isna(x)else x)
 df['GameWeather_dense'] = df['GameWeather_process'].apply(map_weather)
 add_new_feas.append('GameWeather_dense')
 df["Orientation_ob"] = df["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation"].apply(lambda x : np.sin(x / 360 * 2 * np.pi))
df["Orientation_cos"] = df["Orientation"].apply(lambda x : np.cos(x / 360 * 2 * np.pi))
df["Dir_sin"] = df["Dir"].apply(lambda x : np.sin(x / 360 * 2 * np.pi))
df["Dir_cos"] = df["Dir"].apply(lambda x : np.cos(x / 360 * 2 * np.pi))
add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
add_new_feas.append("diffScoreBeforePlay")
 static_features = df[df['NflId'] == df['NflIdRusher']][add_new_feas + ['GameId', 'PlayId', 'X', 'Y', 'S', 'A', 'Dis', 'Orientation',
'Dir', 'YardLine', 'Quarter', 'Down', 'Distance',
'DefendersInTheBox']].drop_duplicates()
static_features.fillna(-999, inplace = True)
 return static_features
def combine_features(relative_to_back, player_on_pitch, defense, static, deploy = deploy):
df = pd.merge(relative_to_back, defense, on = ['GameId', 'PlayId'], how = 'inner')
 df = pd.merge(df, static, on = ['GameId', 'PlayId'], how = 'inner')
 df = pd.merge(df, player_on_pitch, on = ['GameId', 'PlayId'], how = 'inner')
 if not deploy :
df = pd.merge(df, outcomes, on = ['GameId', 'PlayId'], how = 'inner')
 return df
yardline = update_yardline(df)
 df = update_orientation(df, yardline)
 crawlling_dist = crawl_dist(df)
 crawlling_second = crawl_sec(df)
 crawlling_velocity_diff = crawlling_velocity(df, crawlling_dist, crawlling_second)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 static_feats = static_features(df)
 basetable = combine_features(rel_back, def_feats, crawlling_velocity_diff, static_feats, deploy = deploy)
 return basetable<feature_engineering>
del siamese_network
gc.collect()
%time train_basetable = create_features(train, False )<prepare_x_and_y>
dataset_for_training = calculate_features_of_texts(
 texts=multilingual_corpus,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
fe=feature_extractor,
batch_size=batch_size_for_siamese 
)
 assert len(dataset_for_training)== 3
for X in range(1, 23157):
X = train_basetable.copy()
yards = train_basetable.Yards
X = X[~X.isin([np.nan, np.inf, - np.inf] ).any(1)]
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
X.drop(['GameId', 'PlayId', 'Yards'], axis = 1, inplace = True )<normalization>
dataset_for_submission = calculate_features_of_texts(
 texts=texts_for_submission,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
fe=feature_extractor,
batch_size=batch_size_for_siamese
)
scaler = StandardScaler()
X = scaler.fit_transform(X )<split>
X_embedded = []
y_embedded = []
split_by_languages = dict()
start_pos = 0
for cur_lang in dataset_for_training:
X_embedded.append(dataset_for_training[cur_lang][0])
 y_embedded.append(dataset_for_training[cur_lang][1])
 split_by_languages[cur_lang] =(
 set() ,
set(range(start_pos, start_pos + dataset_for_training[cur_lang][1].shape[0])) 
)
 start_pos = start_pos + dataset_for_training[cur_lang][1].shape[0]
featured_data_for_training =(
 np.vstack(X_embedded),
np.concatenate(y_embedded)
)
 for cur_lang in dataset_for_training:
indices_for_testing = split_by_languages[cur_lang][1]
indices_for_training = set(range(featured_data_for_training[0].shape[0])) - indices_for_testing
split_by_languages[cur_lang] =(
 np.array(sorted(list(indices_for_training)) , dtype=np.int32),
np.array(sorted(list(indices_for_testing)) , dtype=np.int32)
)
 del indices_for_training, indices_for_testing
featured_data_for_submission = []
identifies_for_submission = []
for cur_lang in dataset_for_submission:
X_embedded.append(dataset_for_submission[cur_lang][0])
 featured_data_for_submission.append(dataset_for_submission[cur_lang][0])
 identifies_for_submission.append(dataset_for_submission[cur_lang][1])
 y_embedded.append(
 np.array(
 [-1 for _ in range(dataset_for_submission[cur_lang][0].shape[0])],
dtype=np.int32 
)
)
 featured_data_for_submission = np.vstack(featured_data_for_submission)
 identifies_for_submission = np.concatenate(identifies_for_submission)
 X_embedded = np.vstack(X_embedded)
 y_embedded = np.concatenate(y_embedded )
train_basetable.drop(['GameId', 'PlayId', 'Yards'], axis = 1, inplace = True)
 X_train, X_val, y_train, y_val = train_test_split(train_basetable, y, test_size = 0.15, random_state = 12345 )<compute_test_metric>
del dataset_for_training, dataset_for_submission
del feature_extractor, xlmroberta_tokenizer
print(X_train, X_val, y_train, y_val )<split>
indices_of_samples = random.sample(
 list(range(featured_data_for_training[1].shape[0])) ,
k=1000 
)
 indices_of_samples += random.sample(
 list(range(featured_data_for_training[1].shape[0], y_embedded.shape[0])) ,
k=1000 
)
 X_embedded = X_embedded[indices_of_samples]
y_embedded = y_embedded[indices_of_samples]
kf = KFold(n_splits = 4)
 for trn_idx, tst_idx in kf.split(X_train):
X_trn, X_vld = X[trn_idx], X[tst_idx]
y_trn, y_vld = y[trn_idx], y[tst_idx]<compute_test_metric>
X_embedded = TSNE(n_components=2, n_jobs=-1, random_state=42 ).fit_transform(X_embedded )
print(X_trn, X_vld, y_trn, y_vld )<import_modules>
del indices_of_negative_classes
del indices_of_positive_classes
del indices_of_unknown_classes
del indices_of_samples
del X_embedded, y_embedded
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size = 10, include_on_batch = False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs = {}):
pass
def on_train_begin(self, logs = {}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs = {}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs = {}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis = 1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis = 1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis = 1 ).sum(axis = 0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s<compute_test_metric>
gc.collect()
tf.keras.backend.clear_session()
if tpu:
tf.tpu.experimental.shutdown_tpu_system(tpu)
 del strategy
tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu )
def get_model(x_tr, y_tr, x_val, y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(512, input_dim = X.shape[1], activation = 'relu' )(inp)
 x = Dropout(0.25 )(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation = 'relu' )(x)
 x = Dropout(0.25 )(x)
 x = BatchNormalization()(x)
 x = Dense(128, activation = 'relu' )(x)
 x = Dropout(0.25 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation = 'softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = [])
 es = EarlyStopping(monitor = 'CRPS_score_val',
mode = 'min',
restore_best_weights = True,
verbose = 1,
patience = 10)
 mc = ModelCheckpoint('best_model.h5',monitor = 'CRPS_score_val',mode = 'min',
save_best_only = True, verbose = 1, save_weights_only = True)
 bsz = 512
steps = x_tr.shape[0] / bsz
model.fit(x_tr, y_tr, callbacks = [CRPSCallback(validation =(x_val, y_val)) ,es,mc], epochs = 100, batch_size = bsz,verbose = 1)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis = 1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis = 1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis = 1 ).sum(axis = 0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 5)
 return model,crps<split>
splitted_data_for_training = []
for cur_lang in all_languages:
datasets = build_datasets_for_classifier(
 data_for_training=featured_data_for_training,
data_split=split_by_languages,
language_for_testing=cur_lang 
)
 splitted_data_for_training.append(
 {
'train': datasets[0],
'test': datasets[1]
} 
)
 del datasets
losses = []
models = []
crps_csv = []
s_time = time.time()
for k in range(4):
kfold = KFold(20, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(X)) :
print("-----------")
 print("-----------")
 tr_x, tr_y = X[tr_inds], y[tr_inds]
val_x, val_y = X[val_inds], y[val_inds]
model,crps = get_model(tr_x, tr_y, val_x, val_y)
 models.append(model)
 print("the %d fold crps is %f" %(( k_fold + 1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k == 0 :
y_pred = m.predict(x_te,batch_size = 512)
 else:
y_pred += m.predict(x_te,batch_size = 512)
 y_pred = y_pred / model_num
return y_pred<compute_test_metric>
experiment_duration = int(round(time.time() - experiment_start_time))
print('Duration of siamese XLM-RoBERTa preparing is {0} seconds.'.format(
 experiment_duration))
print("mean crps is %f"%np.mean(crps_csv))<predict_on_test>
bnn_params = select_best_model(
 data_for_training=splitted_data_for_training,
languages_for_training=all_languages,
current_strategy=strategy, tpu_system=tpu,
feature_vector_size=featured_data_for_training[0].shape[1],
hidden_layer_size=hidden_layer_of_cls,
batch_size=batch_size_for_cls, max_iters=max_iters_of_cls,
n_monte_carlo=automl_num_monte_carlo,
tmp_file_name=tmp_cls_name
)
%%time
if TRAIN_OFFLINE == False:
env = nflrush.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
basetable = create_features(test_df, deploy = True)
 basetable.drop(['GameId','PlayId'], axis = 1, inplace = True)
 scaled_basetable = scaler.transform(basetable)
 y_pred = predict(scaled_basetable)
 y_pred = np.clip(np.cumsum(y_pred, axis = 1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data = [y_pred], columns = sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()<set_options>
del splitted_data_for_training
TRAIN_ABLE_FALSE=True
if TRAIN_ABLE_FALSE:
os.environ['CUDA_VISIBLE_DEVICES'] = "1"
TRAIN_OFFLINE = False
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )<load_from_csv>
if os.path.isfile(tmp_cls_name):
os.remove(tmp_cls_name )
if TRAIN_OFFLINE:
train = pd.read_csv('.. /input/train.csv', dtype={'WindSpeed': 'object'})
 else:
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})
<remove_duplicates>
n_total_train_samples = featured_data_for_training[1].shape[0]
n_steps_per_epoch = int(np.ceil(n_total_train_samples / float(batch_size_for_cls)) )
outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()<categorify>
with strategy.scope() :
final_bayesian_classifier = build_classifier(
 feature_vector_size=featured_data_for_training[0].shape[1],
n_train_samples=max_iters_of_cls,
hidden_layer_size=hidden_layer_of_cls,
n_hidden_layers=bnn_params['neural_depth'],
C=bnn_params['C'], language='multilang', verbose=True
)
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def strtofloat(x):
try:
return float(x)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def preprocess(train):
train['GameClock_sec'] = train['GameClock'].apply(strtoseconds)
 train["GameClock_minute"] = train["GameClock"].apply(lambda x : x.split(":")[0] ).astype("object")
 train['PlayerHeight_dense'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train["PlayerAge_ob"] = train['PlayerAge'].astype(np.int ).astype("object")
 train['WindSpeed_ob'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train['WindSpeed_dense'] = train['WindSpeed_ob'].apply(strtofloat)
 train['GameWeather_process'] = train['GameWeather'].str.lower()
train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather_dense'] = train['GameWeather_process'].apply(map_weather)
 train['IsRusher'] =(train['NflId'] == train['NflIdRusher'])
 train['IsRusher_ob'] =(train['NflId'] == train['NflIdRusher'] ).astype("object")
 temp = train[train["IsRusher"]][["Team", "PlayId"]].rename(columns={"Team":"RusherTeam"})
 train = train.merge(temp, on = "PlayId")
 train["IsRusherTeam"] = train["Team"] == train["RusherTeam"]
train["Quarter_ob"] = train["Quarter"].astype("object")
 train["Down_ob"] = train["Down"].astype("object")
 train["JerseyNumber_ob"] = train["JerseyNumber"].astype("object")
 train["YardLine_ob"] = train["YardLine"].astype("object")
 train["Orientation_ob"] = train["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Dir_ob"] = train["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Orientation_sin"] = train["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Orientation_cos"] = train["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["Dir_sin"] = train["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Dir_cos"] = train["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["diffScoreBeforePlay"] = train["HomeScoreBeforePlay"] - train["VisitorScoreBeforePlay"]
train["diffScoreBeforePlay_binary_ob"] =(train["HomeScoreBeforePlay"] > train["VisitorScoreBeforePlay"] ).astype("object")
 Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural', 'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 temp = train["OffensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(OffensePersonnelSplit(x)))
 temp.columns = ["Offense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 temp = train["DefensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(DefensePersonnelSplit(x)))
 temp.columns = ["Defense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 train = train.sort_values(by = ['X'] ).sort_values(by = ['Dis'] ).sort_values(by=['PlayId', 'IsRusherTeam', 'IsRusher'] ).reset_index(drop = True)
 return train<prepare_x_and_y>
train_classifier(
 trainset=tf.data.Dataset.from_tensor_slices(
 featured_data_for_training
).repeat().shuffle(n_total_train_samples ).batch(batch_size_for_cls),
n_steps=n_steps_per_epoch,
bayesian_classifier=final_bayesian_classifier, verbose=True,
max_epochs=int(np.ceil(max_iters_of_cls / float(n_total_train_samples))),\
tmp_file_name=tmp_cls_name
)
def create_features(df, deploy=False):
def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0
def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')
 return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
def static_features(df):
add_new_feas = []
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
add_new_feas.append('PlayerHeight_dense')
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] =df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 add_new_feas.append('PlayerAge')
 df['WindSpeed_ob'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed_dense'] = df['WindSpeed_ob'].apply(strtofloat)
 add_new_feas.append('WindSpeed_dense')
 df['GameWeather_process'] = df['GameWeather'].str.lower()
df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather_dense'] = df['GameWeather_process'].apply(map_weather)
 add_new_feas.append('GameWeather_dense')
 df["Orientation_ob"] = df["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Orientation_cos"] = df["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
df["Dir_sin"] = df["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Dir_cos"] = df["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
add_new_feas.append("diffScoreBeforePlay")
 static_features = df[df['NflId'] == df['NflIdRusher']][add_new_feas+['GameId','PlayId','X','Y','S','A','Dis','Orientation','Dir',
'YardLine','Quarter','Down','Distance','DefendersInTheBox']].drop_duplicates()
static_features.fillna(-999,inplace=True)
 return static_features
def combine_features(relative_to_back, defense, static, deploy=deploy):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 if not deploy:
df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')
 return df
yardline = update_yardline(df)
 df = update_orientation(df, yardline)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 static_feats = static_features(df)
 basetable = combine_features(rel_back, def_feats, static_feats, deploy=deploy)
 return basetable<feature_engineering>
result_of_submission = predict_with_model(
 classifier=final_bayesian_classifier,
input_data=featured_data_for_submission,
batch_size=batch_size_for_cls,
n_monte_carlo=final_num_monte_carlo
)
%time train_basetable = create_features(train, False )<prepare_x_and_y>
with codecs.open('submission.csv', mode='w', encoding='utf-8', errors='ignore')as fp:
fp.write('id,toxic
')
 for sample_idx in range(identifies_for_submission.shape[0]):
id_val = identifies_for_submission[sample_idx]
proba_val = result_of_submission[sample_idx]
fp.write('{0},{1:.9f}
'.format(id_val, proba_val))
<normalization><EOS> 
print('Experiment duration is {0:.3f}.'.format(time.time() - experiment_start_time))