Kaggle competition: Predict Future Sales
x_train = fast_encode_xlm(train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode_xlm(valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode_xlm(test.content.astype(str), tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<categorify>
%matplotlib inline
olid_train_encode = fast_encode_xlm(olid_train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_1k_encode = fast_encode_xlm(olid_train_1k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_2k_encode = fast_encode_xlm(olid_train_2k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_5k_encode = fast_encode_xlm(olid_train_5k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_test_encode = fast_encode_xlm(olid_test.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
<prepare_x_and_y>
items = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/items.csv')
 shops = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/shops.csv')
 categories = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/item_categories.csv')
 train = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/test.csv')
 submission = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/sample_submission.csv')
 train.head()
y_olid_train = olid_train.toxic.values
y_olid_train_1k = olid_train_1k.toxic.values
y_olid_train_2k = olid_train_2k.toxic.values
y_olid_train_5k = olid_train_5k.toxic.values
y_olid_test = olid_test.toxic.values<define_variables>
train['item_id'].value_counts(ascending = False)[:5]
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<categorify>
items.loc[items.item_id == 20949]
olid_test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(olid_test_encode)
.batch(BATCH_SIZE)
)
 olid_test_dataset_wlabel =(
 tf.data.Dataset 
.from_tensor_slices(( olid_test_encode, y_olid_test)) 
.batch(BATCH_SIZE)
)
 olid_train_1k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_1k_encode, y_olid_train_1k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_2k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_2k_encode, y_olid_train_2k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_5k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_5k_encode, y_olid_train_5k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_encode, y_olid_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
<choose_model_class>
categories.loc[categories.item_category_id == 71]
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_pretrained>
train['item_cnt_day'].sort_values(ascending=False)[:5]
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFAutoModelWithLMHead 
.from_pretrained('jplu/tf-xlm-roberta-large')
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary() 
<train_model>
train[train.item_cnt_day == 2169]
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
items[items.item_id == 11373]
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
train = train[train.item_cnt_day < 2000]
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False)
<import_modules>
train['item_price'].sort_values(ascending = False)[:5]
from sklearn.metrics import roc_auc_score<save_to_csv>
train[train.item_price == 307980]
olid_test.toxic_predict = model.predict(olid_test_dataset, verbose=1)
 olid_test.to_csv('olid_test_0shot.csv', index=False)
 roc_auc_score(y_true = olid_test.toxic, y_score = olid_test.toxic_predict )<compute_train_metric>
items[items.item_id == 6066]
def Find_Optimal_Cutoff(target, predicted):

 fpr, tpr, threshold = roc_curve(target, predicted)
 i = np.arange(len(tpr))
roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})
 roc_t = roc.iloc[(roc.tf-0 ).abs().argsort() [:1]]
return list(roc_t['threshold'])
 threshold = Find_Optimal_Cutoff(olid_test.toxic, olid_test.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test.toxic_predict]<compute_test_metric>
train[train.item_id == 6066]
f1_score(y_true = olid_test.toxic, y_pred = olid_test.toxic_predict_binary)
<save_to_csv>
train = train[train.item_price < 300000]
model_1k = model
model_1k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_1k.shape[0] // BATCH_SIZE
train_history_1k = model_1k.fit(
 olid_train_1k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_test_dataset_wlabel,
epochs=10 
)
 olid_test1k = olid_test
olid_test1k.toxic_predict = model_1k.predict(olid_test_dataset, verbose=1)
 olid_test1k.to_csv('olid_test_1k.csv', index=False)
 print('1k roc is ' + str(roc_auc_score(y_true = olid_test1k.toxic, y_score = olid_test1k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test1k.toxic, olid_test1k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test1k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test1k.toxic_predict]
print('1k f1-score is ' + str(f1_score(y_true = olid_test1k.toxic, y_pred = olid_test1k.toxic_predict_binary)))
 del model_1k<save_to_csv>
train['item_price'].sort_values() [:5]
model_2k = model
model_2k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_2k.shape[0] // BATCH_SIZE
train_history_2k = model_2k.fit(
 olid_train_2k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_test_dataset_wlabel,
epochs=10 
)
 olid_test2k = olid_test
olid_test2k.toxic_predict = model_2k.predict(olid_test_dataset, verbose=1)
 olid_test2k.to_csv('olid_test_2k.csv', index=False)
 print('2k roc is ' + str(roc_auc_score(y_true = olid_test2k.toxic, y_score = olid_test2k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test2k.toxic, olid_test2k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test2k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test2k.toxic_predict]
print('2k f1-score is ' + str(f1_score(y_true = olid_test2k.toxic, y_pred = olid_test2k.toxic_predict_binary)))
 del model_2k<save_to_csv>
train[train.item_price == -1]
model_5k = model
model_5k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_5k.shape[0] // BATCH_SIZE
train_history_5 = model_5k.fit(
 olid_train_5k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_test_dataset_wlabel,
epochs=10 
)
 olid_test5k = olid_test
olid_test5k.toxic_predict = model_5k.predict(olid_test_dataset, verbose=1)
 olid_test5k.to_csv('olid_test_5k.csv', index=False)
 print('5k roc is ' + str(roc_auc_score(y_true = olid_test5k.toxic, y_score = olid_test5k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test5k.toxic, olid_test5k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test5k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test5k.toxic_predict]
print('5k f1-score is ' + str(f1_score(y_true = olid_test5k.toxic, y_pred = olid_test5k.toxic_predict_binary)))
 del model_5k<save_to_csv>
price_correction = train[(train.shop_id == 32)&(train.item_id == 2973)&(train.date_block_num ==4)&(train.item_price > 0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = price_correction
model_10k = model
model_10k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train.shape[0] // BATCH_SIZE
train_history_10 = model_10k.fit(
 olid_train_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_test_dataset_wlabel,
epochs=10 
)
 olid_test10k = olid_test
olid_test10k.toxic_predict = model_10k.predict(olid_test_dataset, verbose=1)
 olid_test10k.to_csv('olid_test_10k.csv', index=False)
 print('10k roc is ' + str(roc_auc_score(y_true = olid_test10k.toxic, y_score = olid_test10k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test10k.toxic, olid_test10k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test10k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test10k.toxic_predict]
print('10k f1-score is ' + str(f1_score(y_true = olid_test10k.toxic, y_pred = olid_test10k.toxic_predict_binary)))
 del model_10k<import_modules>
shops_train = train.shop_id.nunique()
shops_test = test.shop_id.nunique()
print("Shops in training set = ", shops_train)
 print("Shops in test set = ", shops_test )
import codecs
import copy
import csv
import gc
import os
import pickle
import random
import time
from typing import Dict, List, Sequence, Set, Tuple<import_modules>
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0] )
import matplotlib.pyplot as plt
import numpy as np
from sklearn.manifold import TSNE
from sklearn.metrics import roc_auc_score, roc_curve
import tensorflow as tf
from tensorflow.python.framework import ops, tensor_util
from tensorflow.python.keras.utils import losses_utils, tf_utils
from tensorflow.python.ops import math_ops
from tensorflow.python.ops.losses import util as tf_losses_util
import tensorflow_addons as tfa
from transformers import AutoTokenizer, XLMRobertaTokenizer
from transformers import TFXLMRobertaModel, XLMRobertaConfig<choose_model_class>
LE = preprocessing.LabelEncoder()
LE.fit_transform(shops['city'] )
class LossFunctionWrapper(tf.keras.losses.Loss):
def __init__(self,
fn,
reduction=losses_utils.ReductionV2.AUTO,
name=None,
**kwargs):
super(LossFunctionWrapper, self ).__init__(reduction=reduction, name=name)
 self.fn = fn
self._fn_kwargs = kwargs
def call(self, y_true, y_pred):
if tensor_util.is_tensor(y_pred)and tensor_util.is_tensor(y_true):
y_pred, y_true = tf_losses_util.squeeze_or_expand_dimensions(y_pred, y_true)
 return self.fn(y_true, y_pred, **self._fn_kwargs)
 def get_config(self):
config = {}
for k, v in six.iteritems(self._fn_kwargs):
config[k] = tf.keras.backend.eval(v)if tf_utils.is_tensor_or_variable(v)\
else v
base_config = super(LossFunctionWrapper, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<compute_test_metric>
shops['city_label'] = LE.fit_transform(shops['city'])
 shops.drop(['shop_name', 'city'], axis = 1, inplace = True)
 shops.head()
def distance_based_log_loss(y_true, y_pred):
y_pred = ops.convert_to_tensor(y_pred)
 y_true = math_ops.cast(y_true, y_pred.dtype)
 margin = 1.0
p =(1.0 + tf.math.exp(-margin)) /(1.0 + tf.math.exp(y_pred - margin))
return tf.keras.backend.binary_crossentropy(target=y_true, output=p )<choose_model_class>
len(set(items_test_list)-set(items_train_list))
class DBLLogLoss(LossFunctionWrapper):
def __init__(self, reduction=losses_utils.ReductionV2.AUTO,
name='distance_based_log_loss'):
super(DBLLogLoss, self ).__init__(distance_based_log_loss, name=name,
reduction=reduction )<compute_test_metric>
LE = preprocessing.LabelEncoder()
category_split = categories['item_category_name'].str.split('-')
 categories['main_categories_id'] = category_split.map(lambda row: row[0].strip())
 categories['main_categories_id'] = LE.fit_transform(categories['main_categories_id'])
 categories['sub_category_id'] = category_split.map(lambda row: row[1].strip() if len(row)> 1 else row[0].strip())
 categories['sub_category_id'] = LE.fit_transform(categories['sub_category_id'])
 categories.head()
def dice_loss(y_true, y_pred):
y_pred = ops.convert_to_tensor(y_pred)
 y_true = math_ops.cast(y_true, y_pred.dtype)
 dice_loss_1 = tf.keras.backend.mean(y_true * y_pred, axis=-1)
 dice_loss_2 = tf.keras.backend.mean(y_true * y_true, axis=-1)
 dice_loss_3 = tf.keras.backend.mean(y_pred * y_pred, axis=-1)
 return 1.0 -(2 * dice_loss_1 + 1.0)/(dice_loss_2 + dice_loss_3 + 1.0 )<choose_model_class>
train['date'] = pd.to_datetime(train['date'], format = '%d.%m.%Y')
 train.info()
class DiceLoss(LossFunctionWrapper):
def __init__(self, reduction=losses_utils.ReductionV2.AUTO, name='dice_loss'):
super(DiceLoss, self ).__init__(dice_loss, name=name, reduction=reduction )<categorify>
cartesian_test = []
cartesian_test.append(np.array(jan))
cartesian_test.append(np.array(feb))
class MaskCalculator(tf.keras.layers.Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(MaskCalculator, self ).__init__(**kwargs)
 def build(self, input_shape):
super(MaskCalculator, self ).build(input_shape)
 def call(self, inputs, **kwargs):
return tf.keras.backend.permute_dimensions(
 x=tf.keras.backend.repeat(
 x=tf.keras.backend.cast(
 x=tf.keras.backend.greater(
 x=inputs,
y=0 
),
dtype='float32' 
),
n=self.output_dim 
),
pattern=(0, 2, 1)
)
 def compute_output_shape(self, input_shape):
assert len(input_shape)== 1
shape = list(input_shape)
 shape.append(self.output_dim)
 return tuple(shape )<categorify>
cartesian_test = np.vstack(cartesian_test)
 cartesian_test_df = pd.DataFrame(cartesian_test, columns = ['shop_id', 'item_id', 'date_block_num'])
 cartesian_test_df.head()
def regular_encode(texts: List[str], tokenizer: XLMRobertaTokenizer,
maxlen: int)-> Tuple[np.ndarray, np.ndarray]:
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=True,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids']), np.array(enc_di['attention_mask'] )<load_from_csv>
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float16)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
def load_train_set(file_name: str, text_field: str, sentiment_fields: List[str],
lang_field: str)-> Dict[str, List[Tuple[str, int]]]:
assert len(sentiment_fields)> 0, 'List of sentiment fields is empty!'
header = []
line_idx = 1
data_by_lang = dict()
with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore')as fp:
data_reader = csv.reader(fp, quotechar='"', delimiter=',')
 for row in data_reader:
if len(row)> 0:
err_msg = 'File "{0}": line {1} is wrong!'.format(file_name, line_idx)
 if len(header)== 0:
header = copy.copy(row)
 err_msg2 = err_msg + ' Field "{0}" is not found!'.format(text_field)
 assert text_field in header, err_msg2
for cur_field in sentiment_fields:
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(
 cur_field)
 assert cur_field in header, err_msg2
text_field_index = header.index(text_field)
 try:
lang_field_index = header.index(lang_field)
 except:
lang_field_index = -1
indices_of_sentiment_fields = []
for cur_field in sentiment_fields:
indices_of_sentiment_fields.append(header.index(cur_field))
else:
if len(row)== len(header):
text = row[text_field_index].strip()
assert len(text)> 0, err_msg + ' Text is empty!'
if lang_field_index >= 0:
cur_lang = row[lang_field_index].strip()
assert len(cur_lang)> 0, err_msg + ' Language is empty!'
else:
cur_lang = 'en'
max_proba = 0.0
for cur_field_idx in indices_of_sentiment_fields:
try:
cur_proba = float(row[cur_field_idx])
 except:
cur_proba = -1.0
err_msg2 = err_msg + ' Value {0} is wrong!'.format(
 row[cur_field_idx] 
)
 assert(cur_proba >= 0.0)and(cur_proba <= 1.0), err_msg2
if cur_proba > max_proba:
max_proba = cur_proba
new_label = 1 if max_proba >= 0.5 else 0
if cur_lang not in data_by_lang:
data_by_lang[cur_lang] = []
data_by_lang[cur_lang].append(( text, new_label))
if line_idx % 10000 == 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx,
file_name))
line_idx += 1
if line_idx > 0:
if(line_idx - 1)% 10000 != 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx - 1,
file_name))
return data_by_lang<load_from_csv>
x = train.groupby(['shop_id', 'item_id', 'date_block_num'])['item_cnt_day'].sum().rename('item_cnt_month' ).reset_index()
x.head()
def load_test_set(file_name: str, id_field: str, text_field: str,
lang_field: str)-> Dict[str, List[Tuple[str, int]]]:
header = []
line_idx = 1
data_by_lang = dict()
with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore')as fp:
data_reader = csv.reader(fp, quotechar='"', delimiter=',')
 for row in data_reader:
if len(row)> 0:
err_msg = 'File "{0}": line {1} is wrong!'.format(file_name, line_idx)
 if len(header)== 0:
header = copy.copy(row)
 err_msg2 = err_msg + ' Field "{0}" is not found!'.format(text_field)
 assert text_field in header, err_msg2
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(id_field)
 assert id_field in header, err_msg2
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(lang_field)
 assert lang_field in header, err_msg2
id_field_index = header.index(id_field)
 text_field_index = header.index(text_field)
 lang_field_index = header.index(lang_field)
 else:
if len(row)== len(header):
try:
id_value = int(row[id_field_index])
 except:
id_value = -1
err_msg2 = err_msg + ' {0} is wrong ID!'.format(
 row[id_field_index])
 assert id_value >= 0, err_msg2
text = row[text_field_index].strip()
assert len(text)> 0, err_msg + ' Text is empty!'
if lang_field_index >= 0:
cur_lang = row[lang_field_index].strip()
assert len(cur_lang)> 0, err_msg + ' Language is empty!'
else:
cur_lang = 'en'
if cur_lang not in data_by_lang:
data_by_lang[cur_lang] = []
data_by_lang[cur_lang].append(( text, id_value))
if line_idx % 10000 == 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx,
file_name))
line_idx += 1
if line_idx > 0:
if(line_idx - 1)% 10000 != 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx - 1,
file_name))
return data_by_lang<create_dataframe>
new_train = pd.merge(cartesian_df, x, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0)
 new_train['item_cnt_month'] = np.clip(new_train['item_cnt_month'], 0, 20 )
def build_siamese_dataset(texts: Dict[str, List[Tuple[str, int]]],
dataset_size: int, tokenizer: XLMRobertaTokenizer,
maxlen: int, batch_size: int,
for_training: bool)-> Tuple[tf.data.Dataset, int]:
language_pairs = set()
for language in texts.keys() :
for other_language in texts:
if other_language == language:
language_pairs.add(( language, other_language))
else:
new_pair =(language, other_language)
 new_pair_2 =(other_language, language)
 if(new_pair not in language_pairs)and(new_pair_2 not in language_pairs):
language_pairs.add(new_pair)
 language_pairs = sorted(list(language_pairs))
print('Possible language pairs are: {0}.'.format(language_pairs))
err_msg = '{0} is too small size of the data set!'.format(dataset_size)
 assert dataset_size >=(len(language_pairs)* 10), err_msg
n_samples_for_lang_pair = int(np.ceil(dataset_size / float(len(language_pairs))))
text_pairs_and_labels = []
for left_lang, right_lang in language_pairs:
print('{0}-{1}:'.format(left_lang, right_lang))
left_positive_indices = list(filter(
 lambda idx: texts[left_lang][idx][1] > 0, range(len(texts[left_lang])) 
))
left_negative_indices = list(filter(
 lambda idx: texts[left_lang][idx][1] == 0, range(len(texts[left_lang])) 
))
right_positive_indices = list(filter(
 lambda idx: texts[right_lang][idx][1] > 0, range(len(texts[right_lang])) 
))
right_negative_indices = list(filter(
 lambda idx: texts[right_lang][idx][1] == 0, range(len(texts[right_lang])) 
))
used_pairs = set()
number_of_samples = 0
for _ in range(n_samples_for_lang_pair // 4):
left_idx = random.choice(left_positive_indices)
 right_idx = random.choice(right_positive_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_positive_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
1,
1,
1 
)
)
 number_of_samples += 1
print(' number of "1-1" pairs is {0};'.format(number_of_samples))
number_of_samples = 0
for _ in range(n_samples_for_lang_pair // 4,(2 * n_samples_for_lang_pair)// 4):
left_idx = random.choice(left_negative_indices)
 right_idx = random.choice(right_negative_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_negative_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
1,
0,
0 
)
)
 number_of_samples += 1
print(' number of "0-0" pairs is {0};'.format(number_of_samples))
number_of_samples = 0
for _ in range(( 2 * n_samples_for_lang_pair)// 4, n_samples_for_lang_pair):
left_idx = random.choice(left_negative_indices)
 right_idx = random.choice(right_positive_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_positive_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
if random.random() >= 0.5:
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
0,
0,
1 
)
)
 else:
text_pairs_and_labels.append(
(
 texts[right_lang][right_idx][0],
texts[left_lang][left_idx][0],
0,
1,
0 
)
)
 number_of_samples += 1
print(' number of "0-1" or "1-0" pairs is {0}.'.format(number_of_samples))
random.shuffle(text_pairs_and_labels)
 n_steps = len(text_pairs_and_labels)// batch_size
print('Samples number of the data set is {0}.'.format(len(text_pairs_and_labels)))
 print('Samples number per each language pair is {0}.'.format(n_samples_for_lang_pair))
tokens_of_left_texts, mask_of_left_texts = regular_encode(
 texts=[cur[0] for cur in text_pairs_and_labels],
tokenizer=tokenizer, maxlen=maxlen 
)
 tokens_of_right_texts, mask_of_right_texts = regular_encode(
 texts=[cur[1] for cur in text_pairs_and_labels],
tokenizer=tokenizer, maxlen=maxlen 
)
 siamese_labels = np.array([cur[2] for cur in text_pairs_and_labels], dtype=np.int32)
 left_labels = np.array([cur[3] for cur in text_pairs_and_labels], dtype=np.int32)
 right_labels = np.array([cur[4] for cur in text_pairs_and_labels], dtype=np.int32)
 print('Number of positive siamese samples is {0} from {1}.'.format(
 int(sum(siamese_labels)) , siamese_labels.shape[0]))
print('Number of positive left samples is {0} from {1}.'.format(
 int(sum(left_labels)) , left_labels.shape[0]))
print('Number of positive right samples is {0} from {1}.'.format(
 int(sum(right_labels)) , right_labels.shape[0]))
dataset = tf.data.Dataset.from_tensor_slices(
(
(
 tokens_of_left_texts, mask_of_left_texts,
tokens_of_right_texts, mask_of_right_texts 
), 
(
 siamese_labels,
left_labels,
right_labels 
)
)
 ).batch(batch_size)
 if for_training:
dataset = dataset.repeat()
del text_pairs_and_labels
return dataset, n_steps<choose_model_class>
del x
del cartesian_df
del cartesian
del cartesian_test
del cartesian_test_df
del feb
del jan
del items_test_list
del items_train_list
del train
def build_feature_extractor(transformer_name: str, hidden_state_size: int,
max_len: int)-> Tuple[tf.keras.Model, tf.keras.Model]:
word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="base_word_ids")
 attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="base_attention_mask")
 transformer_layer = TFXLMRobertaModel.from_pretrained(
 pretrained_model_name_or_path=transformer_name,
name='Transformer' 
)
 sequence_output = transformer_layer([word_ids, attention_mask])[0]
output_mask = MaskCalculator(
 output_dim=hidden_state_size, trainable=False,
name='OutMaskCalculator'
)(attention_mask)
 masked_sequence_output = tf.keras.layers.Multiply(
 name='OutMaskMultiplicator'
)([output_mask, sequence_output])
 masked_sequence_output = tf.keras.layers.Masking(
 name='OutMasking'
)(masked_sequence_output)
 pooled_output = tf.keras.layers.GlobalAvgPool1D(name='AvePool' )(masked_sequence_output)
 text_embedding = tf.keras.layers.Lambda(
 lambda x: tf.math.l2_normalize(x, axis=1),
name='Emdedding'
)(pooled_output)
 cls_layer = tf.keras.layers.Dropout(rate=0.3, name='ClsDropout' )(pooled_output)
 cls_layer = tf.keras.layers.Dense(
 units=1, activation='sigmoid', use_bias=True,
kernel_initializer=tf.keras.initializers.GlorotNormal(seed=42),
kernel_regularizer=tf.keras.regularizers.l2(l=1e-4),
name='ClsOutput'
)(cls_layer)
 cls_model = tf.keras.Model(
 inputs=[word_ids, attention_mask],
outputs=cls_layer,
name='Classifier' 
)
 cls_model.build(input_shape=[(None, max_len),(None, max_len)])
 fe_model = tf.keras.Model(
 inputs=[word_ids, attention_mask],
outputs=text_embedding,
name='FeatureExtractor' 
)
 fe_model.build(input_shape=[(None, max_len),(None, max_len)])
 return cls_model, fe_model<compute_test_metric>
new_train.sort_values(['date_block_num','shop_id','item_id'], inplace = True)
 new_train.head()
def euclidean_distance(vects):
x, y = vects
sum_square = tf.keras.backend.sum(tf.keras.backend.square(x - y),
axis=1, keepdims=True)
 return tf.keras.backend.sqrt(
 tf.keras.backend.maximum(sum_square, tf.keras.backend.epsilon())
 )<concatenate>
test.insert(loc=3, column='date_block_num', value=34)
 test['item_cnt_month'] = 0
test.head()
def eucl_dist_output_shape(shapes):
shape1, shape2 = shapes
return(shape1[0], 1 )<choose_model_class>
new_train = new_train.append(test.drop('ID', axis = 1))
def build_siamese_nn(transformer_name: str, hidden_state_size: int, max_len: int,
max_lr: float, base_lr: float, steps_per_epoch: int)-> \
Tuple[tf.keras.Model, tf.keras.Model, tf.keras.Model]:
left_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="left_word_ids")
 left_attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="left_attention_mask")
 right_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="right_word_ids")
 right_attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="right_attention_mask")
 cls_, fe_ = build_feature_extractor(transformer_name, hidden_state_size, max_len)
 left_text_embedding = fe_([left_word_ids, left_attention_mask])
 right_text_embedding = fe_([right_word_ids, right_attention_mask])
 distance_layer = tf.keras.layers.Lambda(
 function=euclidean_distance,
output_shape=eucl_dist_output_shape,
name='L2DistLayer'
)([left_text_embedding, right_text_embedding])
 left_cls_layer = cls_([left_word_ids, left_attention_mask])
 right_cls_layer = cls_([right_word_ids, right_attention_mask])
 nn = tf.keras.Model(
 inputs=[left_word_ids, left_attention_mask, right_word_ids, right_attention_mask],
outputs=[distance_layer, left_cls_layer, right_cls_layer],
name='SiameseXLMR' 
)
 nn.compile(
 optimizer=tf.keras.optimizers.Adam(
 learning_rate=tfa.optimizers.Triangular2CyclicalLearningRate(
 initial_learning_rate=base_lr,
maximal_learning_rate=max_lr,
step_size=int(round(steps_per_epoch * 0.75)) 
)
),
loss=[
DBLLogLoss() ,
tf.keras.losses.BinaryCrossentropy() ,
tf.keras.losses.BinaryCrossentropy()
],
loss_weights=[
1.0,
0.3,
0.3
] 
)
 fe_.summary()
nn.summary()
return nn, cls_, fe_<choose_model_class>
new_train = pd.merge(new_train, shops, on=['shop_id'], how='left')
 new_train.head()
def build_classifier(config: XLMRobertaConfig, hidden_state_size: int, max_len: int,
max_lr: float, base_lr: float, steps_per_epoch: int,
language: str)-> tf.keras.Model:
word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="word_ids_{0}".format(language))
attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="attention_mask_{0}".format(language))
transformer_layer = TFXLMRobertaModel(
 config=config,
name='Transformer_{0}'.format(language.title())
)
 sequence_output = transformer_layer([word_ids, attention_mask])[0]
output_mask = MaskCalculator(
 output_dim=hidden_state_size, trainable=False,
name='OutMaskCalculator_{0}'.format(language)
 )(attention_mask)
 masked_sequence_output = tf.keras.layers.Multiply(
 name='OutMaskMultiplicator_{0}'.format(language)
 )([output_mask, sequence_output])
 masked_sequence_output = tf.keras.layers.Masking(
 name='OutMasking_{0}'.format(language)
 )(masked_sequence_output)
 pooled_output = tf.keras.layers.GlobalAvgPool1D(
 name='AvePool_{0}'.format(language)
 )(masked_sequence_output)
 cls_layer = tf.keras.layers.Dropout(rate=0.3, name='ClsDropout' )(pooled_output)
 cls_layer = tf.keras.layers.Dense(
 units=1, activation='sigmoid', use_bias=True,
kernel_initializer=tf.keras.initializers.GlorotNormal(seed=42),
kernel_regularizer=tf.keras.regularizers.l2(l=1e-4),
name='ClsOutput_{0}'.format(language.title())
 )(cls_layer)
 cls_model = tf.keras.Model(
 inputs=[word_ids, attention_mask],
outputs=cls_layer,
name='Classifier_{0}'.format(language.title())
)
 cls_model.compile(
 optimizer=tf.keras.optimizers.Adam(
 learning_rate=tfa.optimizers.Triangular2CyclicalLearningRate(
 initial_learning_rate=base_lr,
maximal_learning_rate=max_lr,
step_size=int(round(steps_per_epoch * 2.0)) 
)
),
loss=DiceLoss() ,
metrics=[tf.keras.metrics.AUC() ] 
)
 cls_model.summary()
return cls_model<train_model>
new_train = pd.merge(new_train, items.drop('item_name', axis = 1), on=['item_id'], how='left')
 new_train.head()
def train_siamese_nn(nn: tf.keras.Model, trainset: tf.data.Dataset, steps_per_trainset: int,
validset: tf.data.Dataset, steps_per_validset: int,
max_duration: int, model_weights_path: str):
assert steps_per_validset >= 2
assert steps_per_trainset >= 10 * steps_per_validset
steps_per_epoch = 10 * steps_per_validset
n_epochs =(3 * steps_per_trainset)// steps_per_epoch
print('Maximal duration of the Siamese NN training is {0} seconds.'.format(max_duration))
print('n_epochs = {0}, steps_per_epoch = {1}'.format(n_epochs, steps_per_epoch))
print('steps_per_trainset = {0}, steps_per_validset = {1}'.format(
 steps_per_trainset, steps_per_validset 
))
callbacks = [
tfa.callbacks.TimeStopping(seconds=max_duration, verbose=True),
tf.keras.callbacks.ModelCheckpoint(model_weights_path, monitor='val_loss',
mode='min', save_best_only=True,
save_weights_only=True, verbose=True)
 ]
history = nn.fit(
 trainset,
steps_per_epoch=steps_per_epoch,
validation_data=validset,
epochs=n_epochs, callbacks=callbacks 
)
 show_training_process(history, 'loss' )<define_variables>
new_train = pd.merge(new_train, categories.drop('item_category_name', axis = 1), on=['item_category_id'], how='left')
 new_train.head()
def calculate_features_of_texts(texts: Dict[str, List[Tuple[str, int]]],
tokenizer: XLMRobertaTokenizer, maxlen: int,
fe: tf.keras.Model, batch_size: int,
max_dataset_size: int = 0)-> \
Dict[str, Tuple[np.ndarray, np.ndarray]]:
languages = sorted(list(texts.keys()))
datasets_by_languages = dict()
if max_dataset_size > 0:
max_size_per_lang = max_dataset_size // len(languages)
 err_msg = '{0} is too small number of dataset samples!'.format(max_dataset_size)
 assert max_size_per_lang > 0, err_msg
else:
max_size_per_lang = 0
for cur_lang in languages:
selected_indices = list(range(len(texts[cur_lang])))
 if max_size_per_lang > 0:
if len(selected_indices)> max_size_per_lang:
selected_indices = random.sample(
 population=selected_indices,
k=max_size_per_lang 
)
 tokens_of_texts, mask_of_texts = regular_encode(
 texts=[texts[cur_lang][idx][0] for idx in selected_indices],
tokenizer=tokenizer, maxlen=maxlen 
)
 X = []
n_batches = int(np.ceil(len(selected_indices)/ float(batch_size)))
 for batch_idx in range(n_batches):
batch_start = batch_idx * batch_size
batch_end = min(len(selected_indices), batch_start + batch_size)
 res = fe.predict_on_batch(
 [
tokens_of_texts[batch_start:batch_end],
mask_of_texts[batch_start:batch_end]
] 
)
 if not isinstance(res, np.ndarray):
res = res.numpy()
X.append(res)
 del res
X = np.vstack(X)
 y = np.array([texts[cur_lang][idx][1] for idx in selected_indices], dtype=np.int32)
 datasets_by_languages[cur_lang] =(X, y)
 del X, y, selected_indices
return datasets_by_languages<predict_on_test>
def generate_lag(train, months, lag_column):
for month in months:
train_shift = train[['date_block_num', 'shop_id', 'item_id', lag_column]].copy()
train_shift.columns = ['date_block_num', 'shop_id', 'item_id', lag_column+'_lag_'+ str(month)]
train_shift['date_block_num'] += month
train = pd.merge(train, train_shift, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 return train
def predict_with_model(classifier: tf.keras.Model,
input_data: Tuple[np.ndarray, np.ndarray],
batch_size: int)-> np.ndarray:
predicted = []
n_batches = int(np.ceil(input_data[0].shape[0] / float(batch_size)))
 for batch_idx in range(n_batches):
batch_start = batch_idx * batch_size
batch_end = min(batch_start + batch_size, input_data[0].shape[0])
 res = classifier.predict_on_batch(
(
 input_data[0][batch_start:batch_end],
input_data[1][batch_start:batch_end] 
)
)
 if not isinstance(res, np.ndarray):
res = res.numpy()
predicted.append(res.flatten())
 del res
return np.concatenate(predicted )<prepare_x_and_y>
del items
del categories
del shops
del test
def do_submit(data_for_training: Dict[str, Tuple[Tuple[np.ndarray, np.ndarray], np.ndarray]],
data_for_submission: Tuple[np.ndarray, np.ndarray],
language_for_validation: str, maxlen: int, base_classifier: tf.keras.Model,
max_duration: int, batch_size: int)-> Tuple[np.ndarray, int]:
assert language_for_validation in data_for_training
print('Duration of the training procedure must be less ' \
'than {0} seconds.'.format(max_duration))
print('')
 start_time = time.time()
X_train1 = []
X_train2 = []
y_train = []
X_val1 = []
X_val2 = []
y_val = []
for cur_lang in sorted(list(data_for_training.keys())) :
if cur_lang == language_for_validation:
X_val1.append(data_for_training[cur_lang][0][0])
 X_val2.append(data_for_training[cur_lang][0][1])
 y_val.append(data_for_training[cur_lang][1])
 else:
X_train1.append(data_for_training[cur_lang][0][0])
 X_train2.append(data_for_training[cur_lang][0][1])
 y_train.append(data_for_training[cur_lang][1])
 X_train =(np.vstack(X_train1), np.vstack(X_train2))
del X_train1, X_train2
y_train = np.concatenate(y_train)
 trainset = tf.data.Dataset.from_tensor_slices(
(X_train, y_train)
 ).repeat().batch(batch_size)
 del X_train
print('Labeled dataset for training contains {0} samples.'.format(y_train.shape[0]))
print('Number of positive samples is {0} from {1}.'.format(
 int(sum(y_train)) , y_train.shape[0]))
print('')
 steps_per_epoch = int(np.ceil(y_train.shape[0] / float(batch_size)))
 del y_train
X_val =(np.vstack(X_val1), np.vstack(X_val2))
del X_val1, X_val2
y_val = np.concatenate(y_val)
 validset = tf.data.Dataset.from_tensor_slices(
(X_val, y_val)
 ).batch(batch_size)
 print('Dataset for validation contains {0} samples.'.format(y_val.shape[0]))
print('Number of positive samples is {0} from {1}.'.format(int(sum(y_val)) ,
y_val.shape[0]))
print('')
 steps_per_validset = int(np.ceil(y_val.shape[0] / float(batch_size)))
 if steps_per_epoch <=(3 * steps_per_validset):
n_epochs = 25
else:
n_epochs =(25 * steps_per_epoch)//(3 * steps_per_validset)
 steps_per_epoch = 3 * steps_per_validset
callbacks = [
tf.keras.callbacks.EarlyStopping(patience=5, monitor='val_auc', mode='max',
restore_best_weights=True, verbose=True),
tfa.callbacks.TimeStopping(seconds=max_duration, verbose=True)
 ]
print('n_epochs = {0}, steps_per_epoch = {1}, val_steps = {2}'.format(
 n_epochs, steps_per_epoch, steps_per_validset))
history = base_classifier.fit(
 trainset,
steps_per_epoch=steps_per_epoch,
validation_data=validset,
epochs=n_epochs, callbacks=callbacks 
)
 postprocessing_start_time = time.time()
show_training_process(history, 'auc', figure_id=1)
 del history, trainset, validset
print('')
 probabilities = predict_with_model(
 classifier=base_classifier,
input_data=X_val,
batch_size=batch_size 
)
 show_roc_auc(y_true=y_val, probabilities=probabilities,
label='language "{0}"'.format(language_for_validation),
figure_id=2)
 del probabilities, y_val, X_val
gc.collect()
probabilities = predict_with_model(
 classifier=base_classifier,
input_data=data_for_submission,
batch_size=batch_size 
)
 assert probabilities.shape[0] == len(identifiers_for_submission)
 gc.collect()
end_time = time.time()
print('Duration of the final submission for language '\
'"{0}" is {1:.3f} seconds'.format(
 language_for_validation, end_time - start_time))
print('Postprocessing duration(after training)is ' \
'{0:.3f} seconds.'.format(end_time - postprocessing_start_time))
return probabilities, int(np.ceil(end_time - postprocessing_start_time))<define_variables>
new_train = downcast_dtypes(new_train )
experiment_start_time = time.time()<define_variables>
gc.collect()
random.seed(42)
 np.random.seed(42)
 tf.random.set_seed(42 )<define_variables>
%%time
new_train = generate_lag(new_train, [1,2,3,4,5,6,12], 'item_cnt_month' )
siamese_learning_rate_max = 1e-5
siamese_learning_rate_min = 5e-6
classifier_learning_rate_max = 2e-6
classifier_learning_rate_min = 6e-7
dataset_dir = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification'
tmp_roberta_name = '/kaggle/working/base_nn.h5'<load_pretrained>
%%time
group = new_train.groupby(['date_block_num', 'item_id'])['item_cnt_month'].mean().rename('item_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'item_id'], how='left')
 new_train = generate_lag(new_train, [1,2,3,6,12], 'item_month_mean')
 new_train.drop(['item_month_mean'], axis=1, inplace=True )
xlmroberta_tokenizer = AutoTokenizer.from_pretrained(model_name)
 xlmroberta_config = XLMRobertaConfig.from_pretrained(model_name)
 print(xlmroberta_config )<train_model>
%%time
group = new_train.groupby(['date_block_num', 'shop_id'])['item_cnt_month'].mean().rename('shop_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'shop_id'], how='left')
 new_train = generate_lag(new_train, [1,2,3,6,12], 'shop_month_mean')
 new_train.drop(['shop_month_mean'], axis=1, inplace=True )
sentence_embedding_size = xlmroberta_config.hidden_size
print('Sentence embedding size is {0}'.format(sentence_embedding_size))
assert max_seq_len <= xlmroberta_config.max_position_embeddings<load_from_csv>
%%time
group = new_train.groupby(['date_block_num', 'shop_id', 'item_category_id'])['item_cnt_month'].mean().rename('shop_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 new_train = generate_lag(new_train, [1, 2], 'shop_category_month_mean')
 new_train.drop(['shop_category_month_mean'], axis=1, inplace=True )
corpus_for_training = load_train_set(
 os.path.join(dataset_dir, "jigsaw-unintended-bias-train.csv"),
text_field="comment_text", lang_field="lang",
sentiment_fields=["toxic", "severe_toxicity", "obscene", "identity_attack", "insult",
"threat"] 
)
 assert 'en' in corpus_for_training<load_from_csv>
%%time
group = new_train.groupby(['date_block_num', 'main_category_id'])['item_cnt_month'].mean().rename('main_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'main_category_id'], how='left')
 new_train = generate_lag(new_train, [1], 'main_category_month_mean')
 new_train.drop(['main_category_month_mean'], axis=1, inplace=True )
corpus_for_validation = load_train_set(
 os.path.join(dataset_dir, "jigsaw-toxic-comment-train.csv"),
text_field="comment_text", lang_field="lang",
sentiment_fields=["toxic", "severe_toxic", "obscene", "threat", "insult",
"identity_hate"] 
)
 assert 'en' in corpus_for_validation<load_from_csv>
%%time
group = new_train.groupby(['date_block_num', 'sub_category_id'])['item_cnt_month'].mean().rename('sub_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'sub_category_id'], how='left')
 new_train = generate_lag(new_train, [1], 'sub_category_month_mean')
 new_train.drop(['sub_category_month_mean'], axis=1, inplace=True )
multilingual_corpus = load_train_set(
 os.path.join(dataset_dir, "validation.csv"),
text_field="comment_text", lang_field="lang", sentiment_fields=["toxic", ] 
)
 assert 'en' not in multilingual_corpus
max_size = 0
print('Multilingual data:')
 for language in sorted(list(multilingual_corpus.keys())) :
print(' {0}\t\t{1} samples'.format(language, len(multilingual_corpus[language])))
 assert set(map(lambda cur: cur[1], multilingual_corpus[language])) == {0, 1}
if len(multilingual_corpus[language])> max_size:
max_size = len(multilingual_corpus[language] )<load_from_csv>
new_train['month'] = new_train['date_block_num'] % 12
texts_for_submission = load_test_set(
 os.path.join(dataset_dir, "test.csv"),
text_field="content", lang_field="lang", id_field="id" 
)
 print('Data for submission:')
 for language in sorted(list(texts_for_submission.keys())) :
print(' {0}\t\t{1} samples'.format(language, len(texts_for_submission[language])) )<create_dataframe>
holiday_dict = {
0: 6,
1: 3,
2: 2,
3: 8,
4: 3,
5: 3,
6: 2,
7: 8,
8: 4,
9: 8,
10: 5,
11: 4,
}
new_train['holidays_in_month'] = new_train['month'].map(holiday_dict )
dataset_for_training, n_train_batches = build_siamese_dataset(
 texts=corpus_for_training, dataset_size=300000,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
batch_size=batch_size_for_siamese, for_training=True
)<drop_column>
moex = {
12: 659, 13: 640, 14: 1231,
15: 881, 16: 764, 17: 663,
18: 743, 19: 627, 20: 692,
21: 736, 22: 680, 23: 1092,
24: 657, 25: 863, 26: 720,
27: 819, 28: 574, 29: 568,
30: 633, 31: 658, 32: 611,
33: 770, 34: 723,
}
new_train['moex_value'] = new_train.date_block_num.map(moex)
 new_train = downcast_dtypes(new_train )
del corpus_for_training<create_dataframe>
gc.collect()
dataset_for_validation, n_val_batches = build_siamese_dataset(
 texts=multilingual_corpus, dataset_size=2000,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
batch_size=batch_size_for_siamese, for_training=False
)<drop_column>
new_train = new_train[new_train.date_block_num > 11]
del corpus_for_validation<set_options>
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
df[col].fillna(0, inplace=True)
 return df
new_train = fill_na(new_train )
gc.collect()<train_model>
def xgtrain() :
regressor = xgb.XGBRegressor(n_estimators = 5000,
learning_rate = 0.01,
max_depth = 10,
subsample = 0.5,
colsample_bytree = 0.5)
 regressor_ = regressor.fit(new_train[new_train.date_block_num < 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num < 33]['item_cnt_month'].values,
eval_metric = 'rmse',
eval_set = [(new_train[new_train.date_block_num < 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num < 33]['item_cnt_month'].values), 
(new_train[new_train.date_block_num == 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num == 33]['item_cnt_month'].values)
 ],
verbose=True,
early_stopping_rounds = 50, 
)
 return regressor_
preparing_duration = int(round(time.time() - experiment_start_time))
print("Duration of data loading and preparing to the Siamese NN training is "
"{0} seconds.".format(preparing_duration))<choose_model_class>
%%time
regressor_ = xgtrain()
with strategy.scope() :
siamese_network, neural_classifier, feature_extractor = build_siamese_nn(
 transformer_name=model_name,
hidden_state_size=sentence_embedding_size,
max_len=max_seq_len,
max_lr=siamese_learning_rate_max,
base_lr=siamese_learning_rate_min,
steps_per_epoch=n_val_batches * 10
)<train_model>
predictions = regressor_.predict(new_train[new_train.date_block_num == 34].drop(['item_cnt_month'], axis = 1 ).values )
<drop_column><EOS> 
submission['item_cnt_month'] = predictions
submission.to_csv('sales_faster_learn.csv', index=False )