Kaggle competition: Predict Future Sales
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=True,
return_token_type_ids=True,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'],dtype=np.int32),np.array(enc_di['attention_mask'],dtype=np.int32),np.array(enc_di['token_type_ids'],dtype=np.int32 )<string_transform>
items=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/items.csv")
 shops=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/shops.csv")
 cats=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/item_categories.csv")
 train=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/sales_train.csv")
 test=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/test.csv" )
LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
def get_sentences(text, lang='en'):
return sent_tokenize(text, LANGS.get(lang, 'english'))
def exclude_duplicate_sentences(text, lang='en'):
sentences = []
for sentence in get_sentences(text, lang):
sentence = sentence.strip()
if sentence not in sentences:
sentences.append(sentence)
 return ' '.join(sentences)
 def clean_text(text, lang='en'):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 text = exclude_duplicate_sentences(text, lang)
 return text.strip()<categorify>
train = train[(train.item_price < 300000)&(train.item_cnt_day < 1000)]
def build_model(transformer, max_len=512):

 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, _, hs = transformer(( input_ids,input_mask,segment_ids))
x = tf.stack([hs[-1],hs[-2],hs[-3]])
 sequence_output = tf.reduce_mean(x, axis = 0)
 pool_output = []
kernel_sizes = [3, 4, 5]
for kernel_size in kernel_sizes:
c = Conv1D(filters=64, kernel_size=kernel_size, padding='same',activation='relu',strides=1 )(sequence_output)
 p = GlobalMaxPooling1D()(c)
 pool_output.append(p)
 pool_output = concatenate([p for p in pool_output])
 out = pool_output
out = Dropout(0.3 )(out)
 out = Dense(64, activation='relu' )(out)
 out = Dropout(0.3 )(out)
 out = Dense(1, activation='sigmoid' )(out)
 model = Model(inputs=[input_ids,input_mask,segment_ids], outputs=out)
 model.compile(Adam(lr=0.2e-5), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.AUC() ])
 return model<set_options>
train = train[train.item_price > 0].reset_index(drop = True)
 train.loc[train.item_cnt_day < 1, "item_cnt_day"] = 0
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 210
MODEL = 'jplu/tf-xlm-roberta-large'
PRETRAINED_MODEL = '/kaggle/input/jigsaw-mlm-finetuned-xlm-r-large'<load_pretrained>
shops.loc[ shops.shop_name == 'Сергиев Посад ТЦ "7Я"',"shop_name" ] = 'СергиевПосад ТЦ "7Я"'
shops["city"] = shops.shop_name.str.split(" " ).map(lambda x: x[0])
 shops["category"] = shops.shop_name.str.split(" " ).map(lambda x: x[1])
 shops.loc[shops.city == "!Якутск", "city"] = "Якутск"
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
shops["shop_category"] = LabelEncoder().fit_transform(shops.category)
 shops["shop_city"] = LabelEncoder().fit_transform(shops.city)
 shops = shops[["shop_id", "shop_category", "shop_city"]]
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<load_from_csv>
cats["type_code"] = cats.item_category_name.apply(lambda x: x.split(" ")[0] ).astype(str)
 cats.loc[(cats.type_code == "Игровые")|(cats.type_code == "Аксессуары"), "category" ] = "Игры"
train3 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv")
 train4 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-it-cleaned.csv")
 train5 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-pt-cleaned.csv")
 train6 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-ru-cleaned.csv")
 train7 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-fr-cleaned.csv")
 train8 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv" )<count_values>
cats.type_code = LabelEncoder().fit_transform(cats.type_code)
 cats["split"] = cats.item_category_name.apply(lambda x: x.split("-"))
cats["subtype"] = cats.split.apply(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats["subtype_code"] = LabelEncoder().fit_transform(cats["subtype"])
 cats = cats[["item_category_id", "subtype_code", "type_code"]]
train1['toxic'].value_counts()<concatenate>
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1' ).sample(n=60000, random_state=12312),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=60000, random_state=42323),
])
 train_tr = pd.concat([
train3[['comment_text', 'toxic']].query('toxic==1'),
train3[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=23412),
])
 train_it = pd.concat([
train4[['comment_text', 'toxic']].query('toxic==1'),
train4[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=23412),
])
 train_pt = pd.concat([
train5[['comment_text', 'toxic']].query('toxic==1'),
train5[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=75293),
])
 train_ru = pd.concat([
train6[['comment_text', 'toxic']].query('toxic==1'),
train6[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=17479),
])
 train_fr = pd.concat([
train7[['comment_text', 'toxic']].query('toxic==1'),
train7[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=56321),
])
 train_es = pd.concat([
train8[['comment_text', 'toxic']].query('toxic==1'),
train8[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=45874),
] )<feature_engineering>
items["name1"], items["name2"] = items.item_name.str.split("[", 1 ).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1 ).str
items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items = items.fillna('0')
 items["item_name"] = items["item_name"].apply(lambda x: name_correction(x))
items.name2 = items.name2.apply(lambda x: x[:-1] if x !="0" else "0" )
train['comment_text'] = train['comment_text'].apply(lambda x : clean_text(x))
valid['comment_text'] = valid['comment_text'].apply(lambda x : clean_text(x))
train_tr['comment_text'] = train_tr['comment_text'].apply(lambda x : clean_text(x,lang='tr'))
train_pt['comment_text'] = train_pt['comment_text'].apply(lambda x : clean_text(x,lang='pt'))
train_it['comment_text'] = train_it['comment_text'].apply(lambda x : clean_text(x,lang='it'))
train_ru['comment_text'] = train_ru['comment_text'].apply(lambda x : clean_text(x,lang='ru'))
train_fr['comment_text'] = train_fr['comment_text'].apply(lambda x : clean_text(x,lang='fr'))
train_es['comment_text'] = train_es['comment_text'].apply(lambda x : clean_text(x,lang='es'))
test['content'] = test['content'].apply(lambda x : clean_text(x)) 
<categorify>
items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0])
 items.loc[(items.type == "x360")|(items.type == "xbox360")|(items.type == "xbox 360"),"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply(lambda x: x.replace(" ", ""))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3"
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_tr = regular_encode(train_tr.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_pt = regular_encode(train_pt.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_it = regular_encode(train_it.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_ru = regular_encode(train_ru.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_fr = regular_encode(train_fr.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_es = regular_encode(train_es.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_train_tr = train_tr.toxic.values
y_train_pt = train_pt.toxic.values
y_train_it = train_it.toxic.values
y_train_ru = train_ru.toxic.values
y_train_fr = train_fr.toxic.values
y_train_es = train_es.toxic.values
y_valid = valid.toxic.values<split>
group_sum = items.groupby(["type"] ).agg({"item_id": "count"})
 group_sum = group_sum.reset_index()
drop_cols = []
for cat in group_sum.type.unique() :
if group_sum.loc[(group_sum.type == cat), "item_id"].values[0] <40:
drop_cols.append(cat)
 items.name2 = items.name2.apply(lambda x: "other" if(x in drop_cols)else x)
 items = items.drop(["type"], axis = 1 )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.shuffle(len(train)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_tr_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_tr, y_train_tr)) 
.shuffle(len(train_tr)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_it_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_it, y_train_it)) 
.shuffle(len(train_it)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_pt_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_pt, y_train_pt)) 
.shuffle(len(train_pt)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_ru_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_ru, y_train_ru)) 
.shuffle(len(train_ru)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_fr_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_fr, y_train_fr)) 
.shuffle(len(train_fr)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_es_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_es, y_train_es)) 
.shuffle(len(train_es)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
<load_pretrained>
items.name2 = LabelEncoder().fit_transform(items.name2)
 items.name3 = LabelEncoder().fit_transform(items.name3)
 items.drop(["item_name", "name1"],axis = 1, inplace= True)
 items.head()
%%time
with strategy.scope() :
config = AutoConfig.from_pretrained(PRETRAINED_MODEL)
 config.output_hidden_states = True
transformer_layer = TFAutoModel.from_pretrained(PRETRAINED_MODEL,config=config)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
ts = time.time()
matrix = []
cols = ["date_block_num", "shop_id", "item_id"]
for i in range(34):
sales = train[train.date_block_num == i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype = np.int16))
matrix = pd.DataFrame(np.vstack(matrix), columns = cols)
 matrix["date_block_num"] = matrix["date_block_num"].astype(np.int8)
 matrix["shop_id"] = matrix["shop_id"].astype(np.int8)
 matrix["item_id"] = matrix["item_id"].astype(np.int16)
 matrix.sort_values(cols, inplace = True)
 time.time() - ts
n_steps = train_tr.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_tr_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS+1,
shuffle=False,
)<train_model>
train["revenue"] = train["item_cnt_day"] * train["item_price"]
n_steps = train_it.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_it_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)<train_model>
ts = time.time()
group = train.groupby(["date_block_num", "shop_id", "item_id"] ).agg({"item_cnt_day": ["sum"]})
 group.columns = ["item_cnt_month"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = cols, how = "left")
 matrix["item_cnt_month"] = matrix["item_cnt_month"].fillna(0 ).astype(np.float16)
 time.time() - ts
n_steps = train_pt.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_pt_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)<train_model>
test["date_block_num"] = 34
test["date_block_num"] = test["date_block_num"].astype(np.int8)
 test["shop_id"] = test.shop_id.astype(np.int8)
 test["item_id"] = test.item_id.astype(np.int16 )
n_steps = train_ru.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_ru_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)<train_model>
ts = time.time()
matrix = pd.concat([matrix, test.drop(["ID"],axis = 1)], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace = True)
 time.time() - ts
n_steps = train_fr.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_fr_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)<train_model>
ts = time.time()
matrix = pd.merge(matrix, shops, on = ["shop_id"], how = "left")
 matrix = pd.merge(matrix, items, on = ["item_id"], how = "left")
 matrix = pd.merge(matrix, cats, on = ["item_category_id"], how = "left")
 matrix["shop_city"] = matrix["shop_city"].astype(np.int8)
 matrix["shop_category"] = matrix["shop_category"].astype(np.int8)
 matrix["item_category_id"] = matrix["item_category_id"].astype(np.int8)
 matrix["subtype_code"] = matrix["subtype_code"].astype(np.int8)
 matrix["name2"] = matrix["name2"].astype(np.int8)
 matrix["name3"] = matrix["name3"].astype(np.int16)
 matrix["type_code"] = matrix["type_code"].astype(np.int8)
 time.time() - ts
n_steps = train_es.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_es_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)<train_model>
def lag_feature(df,lags, cols):
for col in cols:
print(col)
 tmp = df[["date_block_num", "shop_id","item_id",col ]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ["date_block_num", "shop_id", "item_id", col + "_lag_"+str(i)]
shifted.date_block_num = shifted.date_block_num + i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
%%time
n_steps = valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=5,
verbose=2,
shuffle=False,
)<save_to_csv>
ts = time.time()
group = matrix.groupby(["date_block_num"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1], ["date_avg_item_cnt"])
 matrix.drop(["date_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts
sub['toxic'] = model.predict(x_test, verbose=1)
 sub.to_csv('submission.csv', index=False)
 sub.head(10 )<import_modules>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix.date_item_avg_item_cnt = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ['date_item_avg_item_cnt'])
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
import os
import re
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input, Embedding, Conv1D, MaxPooling1D, Flatten
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import transformers
from transformers import TFAutoModel, AutoTokenizer, RobertaTokenizerFast
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors 
<categorify>
ts = time.time()
group = matrix.groupby(["date_block_num","shop_id"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_shop_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num","shop_id"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_shop_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ["date_shop_avg_item_cnt"])
 matrix.drop(["date_shop_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts
def clean_sentence(sentence):

 clean_sent =
for word in sentence.split() :
if len(word)> 1 and not re.match(r'.*[0-9]+', word):
clean_sent = "{} {}".format(clean_sent, word)
 clean_sent = re.sub(r'[.,:;?!/\|@
clean_sent = text = re.sub(r'(http|www)\S*', '', clean_sent)
 return clean_sent.strip()
def clean_df(df, column):

 df = df.apply(lambda x: x.astype(str ).str.lower())
 df[column] = df[column].apply(lambda x: clean_sentence(x))
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
 def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def build_CNN_model() :
input_layer = Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_words_ids")
 embedding = Embedding(VOCAB_SIZE[0], VOCAB_SIZE[1], input_length=MAX_LEN, name='embed' )(input_layer)
 conv_1 = Conv1D(256,(100), activation='relu' )(embedding)
 max_pool = MaxPooling1D()(conv_1)
 conv_1 = Conv1D(128,(5), activation='relu' )(max_pool)
 max_pool = MaxPooling1D()(conv_1)
 dense = Dense(128, activation='relu' )(max_pool)
 dense = Dense(128, activation='relu' )(dense)
 flatten = Flatten()(dense)
 out = Dense(1, activation='sigmoid' )(flatten)
 model = Model(inputs = input_layer, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<set_options>
ts = time.time()
group = matrix.groupby(["date_block_num","shop_id","item_id"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_shop_item_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num","shop_id","item_id"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_shop_item_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ["date_shop_item_avg_item_cnt"])
 matrix.drop(["date_shop_item_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix.date_shop_subtype_avg_item_cnt = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_shop_subtype_avg_item_cnt'])
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 1
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_city_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', "shop_city"], how='left')
 matrix.date_city_avg_item_cnt = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_city_avg_item_cnt'])
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'shop_city'], how='left')
 matrix.date_item_city_avg_item_cnt = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_item_city_avg_item_cnt'])
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train3 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv")
 train4 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-fr-cleaned.csv")
 train5 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-it-cleaned.csv")
 train6 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-pt-cleaned.csv")
 train7 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-ru-cleaned.csv")
 train8 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv")
 valid1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 valid2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv')
 test1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 test2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
<data_type_conversions>
ts = time.time()
group = train.groupby(["item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["item_avg_item_price"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["item_id"], how = "left")
 matrix["item_avg_item_price"] = matrix.item_avg_item_price.astype(np.float16)
 group = train.groupby(["date_block_num","item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["date_item_avg_item_price"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["date_block_num","item_id"], how = "left")
 matrix["date_item_avg_item_price"] = matrix.date_item_avg_item_price.astype(np.float16)
 lags = [1, 2, 3]
matrix = lag_feature(matrix, lags, ["date_item_avg_item_price"])
 for i in lags:
matrix["delta_price_lag_" + str(i)] =(matrix["date_item_avg_item_price_lag_" + str(i)]- matrix["item_avg_item_price"])/ matrix["item_avg_item_price"]
def select_trends(row):
for i in lags:
if row["delta_price_lag_" + str(i)]:
return row["delta_price_lag_" + str(i)]
return 0
matrix["delta_price_lag"] = matrix.apply(select_trends, axis = 1)
 matrix["delta_price_lag"] = matrix.delta_price_lag.astype(np.float16)
 matrix["delta_price_lag"].fillna(0 ,inplace = True)
 features_to_drop = ["item_avg_item_price", "date_item_avg_item_price"]
for i in lags:
features_to_drop.append("date_item_avg_item_price_lag_" + str(i))
features_to_drop.append("delta_price_lag_" + str(i))
matrix.drop(features_to_drop, axis = 1, inplace = True)
 time.time() - ts
clean_df(train, 'comment_text')
 train.toxic = train.toxic.round().astype(int)
 train = train.sample(frac = 1)
 clean_df(valid, 'comment_text')
 clean_df(test1, 'content')
 clean_df(test2, 'content' )<prepare_x_and_y>
ts = time.time()
group = train.groupby(["date_block_num","shop_id"] ).agg({"revenue": ["sum"] })
 group.columns = ["date_shop_revenue"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group , on = ["date_block_num", "shop_id"], how = "left")
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(["shop_id"] ).agg({ "date_block_num":["mean"] })
 group.columns = ["shop_avg_revenue"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["shop_id"], how = "left")
 matrix["shop_avg_revenue"] = matrix.shop_avg_revenue.astype(np.float32)
 matrix["delta_revenue"] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix["delta_revenue"] = matrix["delta_revenue"].astype(np.float32)
 matrix = lag_feature(matrix, [1], ["delta_revenue"])
 matrix["delta_revenue_lag_1"] = matrix["delta_revenue_lag_1"].astype(np.float32)
 matrix.drop(["date_shop_revenue", "shop_avg_revenue", "delta_revenue"] ,axis = 1, inplace = True)
 time.time() - ts
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test1 = regular_encode(test1.content.values, tokenizer, maxlen=MAX_LEN)
 x_test2 = regular_encode(test2.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values 
<categorify>
matrix["month"] = matrix["date_block_num"] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix["days"] = matrix["month"].map(days ).astype(np.int8 )
x_train1 = regular_encode(train1.comment_text.values, tokenizer, maxlen=MAX_LEN)
 train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
<categorify>
ts = time.time()
matrix["item_shop_first_sale"] = matrix["date_block_num"] - matrix.groupby(["item_id","shop_id"])["date_block_num"].transform('min')
 matrix["item_first_sale"] = matrix["date_block_num"] - matrix.groupby(["item_id"])["date_block_num"].transform('min')
 time.time() - ts
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test1_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test1)
.batch(BATCH_SIZE)
)
 test2_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test2)
.batch(BATCH_SIZE)
 )<define_variables>
ts = time.time()
matrix = matrix[matrix["date_block_num"] > 3]
time.time() - ts
VOCAB_SIZE = x_train.shape<choose_model_class>
rcParams['figure.figsize'] = 12, 4
with strategy.scope() :
cnn_model = build_CNN_model()
cnn_model.summary()<train_model>
data = matrix.copy()
del matrix
gc.collect()
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = roberta_model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = roberta_model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<train_model>
del data
gc.collect() ;
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = cnn_model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS*5
)<train_model>
ts = time.time()
model = XGBRegressor(
 max_depth=10,
n_estimators=1000,
min_child_weight=0.5,
colsample_bytree=0.8,
subsample=0.8,
eta=0.1,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 20)
 time.time() - ts
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = cnn_model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<predict_on_test>
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False )