Kaggle competition: Predict Future Sales
multi_ling_sub = roberta_model.predict(test1_dataset, verbose=1)
 eng_sub = roberta_model.predict(test2_dataset, verbose=1)
 sub['toxic'] = multi_ling_sub*0.5 + eng_sub*0.5<save_to_csv>
test = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/test.csv')
 item_cat = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/item_categories.csv')
 items = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/items.csv')
 train = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/sales_train.csv')
 shop = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/shops.csv' )
sub.to_csv('submission.csv', index=False )<save_to_csv>
train['item_id'].value_counts(ascending=False)[:10]
sub.to_csv('submission.csv', index=False )<import_modules>
items.loc[items['item_id']==20949]
import os,re,gc,pickle,random,sys
import numpy as np
import pandas as pd
import tensorflow as tf
import transformers
from transformers import TFAutoModel
from tensorflow.data.experimental import sample_from_datasets
from tensorflow.data import Dataset
import matplotlib.pyplot as plt
import seaborn as sns
from tqdm.notebook import tqdm<define_variables>
train['item_cnt_day'].sort_values(ascending=False)[:10]
np.random.seed(1234)
 random.seed(1234)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
with strategy.scope() :
tf.random.set_seed(1234 )<define_variables>
train[train['item_cnt_day'] == 2169]
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
HEAD = "mean"
input1 = "/kaggle/input/jigsaw-multilingual-toxic-comment-classification/"
inpath = ".. /input/jwtc-xlmroberta-encoding-192-pickle/datain/"
form = "training/encode_{}.pkl"
langs = ["en","en2","es","fr","it","pt","ru","tr"]
used_data = langs<load_pretrained>
train[train['item_cnt_day'] == 1000]
def pick_load_format(path):
with open(inpath+path,"rb")as f:
return pickle.load(f)
 def load(path):
return pick_load_format(form.format(path))<categorify>
items[items['item_id'] == 11373]
def get_cong(n,verb=True):
tot = round(1+(n*2)/10_000)*10_000
if verb: print("Pos: {}, Sample neg: {}, Total: {}".format(n,tot-n,tot))
return tot,tot-n
def load_data(seed=1214):
train = []
for i in used_data:
df1 = load(i+"_l1")
 size, sample_size = get_cong(df1.shape[0])
 df0 = load(i+"_l0" ).sample(n=sample_size, random_state=seed)
 train += [df1,df0]
train = pd.concat(train)
 train = np.stack(train.comment_text.values, axis=0 ).astype("int32"),train.toxic.values
valid = pick_load_format("valid.pkl")
 x_valid = np.stack(valid.comment_text.values, axis=0 ).astype("int32")
 y_valid = valid.toxic.values
test = pick_load_format("test.pkl")
 x_test = np.stack(test.content.values, axis=0 ).astype("int32")
 return train,(x_valid,y_valid),x_test<set_options>
train = train[train['item_cnt_day'] < 2000]
%%time
train,valid,test = load_data()
gc.collect()
valid_size = len(valid[1])
 train_size = len(train[1])
 print(train_size,valid_size )<define_variables>
train['item_price'].sort_values(ascending=False)[:10]
def make_dataset_pipeline(dataset, cache=False,repeat_and_shuffle=False,shuffle_size=128_000,seed=386491):
if cache: dataset = dataset.cache()
if repeat_and_shuffle:
dataset = dataset.repeat().shuffle(shuffle_size,seed)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset<create_dataframe>
train[train['item_price'] == 307980]
train_dataset = Dataset.from_tensor_slices(train)
 train_dataset = make_dataset_pipeline(train_dataset,True, repeat_and_shuffle=True)
 valid_dataset = make_dataset_pipeline(Dataset.from_tensor_slices(valid))
valid_dataset2 = make_dataset_pipeline(Dataset.from_tensor_slices(( valid)) , True,
shuffle_size=8000, repeat_and_shuffle=True)
 test_dataset = make_dataset_pipeline(Dataset.from_tensor_slices(test))<choose_model_class>
items[items['item_id'] == 6066]
def get_cls(x):
return x[:, 0, :]
dic = {"mean":GlobalAveragePooling1D() ,
"max":GlobalMaxPool1D() ,
"cls":get_cls}
def build_model(transformer,head="cls" , loss='binary_crossentropy',
max_len=512, drop_rate=None, lr=1e-5,seed=940208):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
x = dic[head](sequence_output)
 if drop_rate is not None:
x = Dropout(drop_rate )(x)
 out = Dense(1, activation='sigmoid',kernel_initializer=GlorotUniform(seed))(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=lr), loss=loss, metrics=[AUC() ])
 return model<load_pretrained>
train = train[train['item_price'] < 300000]
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN)
 model.summary()<choose_model_class>
train['item_price'].sort_values() [:5]
model_path = "xlm-roberta.h5"
checkpoint = ModelCheckpoint(model_path, monitor='val_auc', mode='max', save_best_only=True, save_weights_only=True, verbose=1)
 es = EarlyStopping(monitor='val_auc', mode='max', patience=6, restore_best_weights=False, verbose=1)
 rp = ReduceLROnPlateau(monitor='val_auc', factor=0.8, patience=3, verbose=1, mode='max')
 callback_list = [checkpoint,es,rp]<train_model>
train[train['item_price'] == -1]
%%time
N_STEPS = train_size //(BATCH_SIZE*4)
 EPOCHS = 10
train_history = model.fit(
 train_dataset,
steps_per_epoch=N_STEPS,
validation_data=valid_dataset,
callbacks=callback_list,
epochs=EPOCHS
)<set_options>
items[items['item_id'] == 2973]
del model
gc.collect()
tf.tpu.experimental.initialize_tpu_system(tpu )<load_pretrained>
price_correction = train[(train['shop_id'] == 32)&(train['item_id'] == 2973)&(train['date_block_num'] == 4)&(train['item_price'] > 0)].item_price.median()
train.loc[train['item_price'] < 0, 'item_price'] = price_correction
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN,lr=8e-6)
 model.load_weights(model_path )<train_model>
train['item_price'].sort_values(ascending=False )
%%time
n_steps = valid_size //(BATCH_SIZE)
 EPOCHS = 1
train_history_2 =model.fit(
 valid_dataset2,
steps_per_epoch=n_steps,
epochs= EPOCHS
)<save_to_csv>
test['item_id'].value_counts(ascending=False)[:5]
sub = pd.read_csv(input1 + "sample_submission.csv")
 sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<import_modules>
shop_train = train['shop_id'].nunique()
shop_test = test['shop_id'].nunique()
import numpy as np
import pandas as pd
from scipy.special import softmax
import os<load_from_csv>
train.loc[train['shop_id'] == 0, 'shop_id'] = 57
test.loc[test['shop_id'] == 0, 'shop_id'] = 57
train.loc[train['shop_id'] == 1, 'shop_id'] = 58
test.loc[test['shop_id'] == 1, 'shop_id'] = 58
train.loc[train['shop_id'] == 10, 'shop_id'] = 11
test.loc[test['shop_id'] == 10, 'shop_id'] = 11
in_path = '/kaggle/input/'
path = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'
files = ['tpu-inference-super-fast-xlmroberta/submission.csv',
'tpu-training-super-fast-xlmroberta/submission.csv',
'train-from-mlm-finetuned-xlm-roberta-large/submission.csv',
'xlm-roberta-augmentation-ssl-0-9417-pub-lb/submission.csv',
'jigsaw-tpu-bert-with-huggingface-and-keras/submission.csv']
sumbs = np.array([pd.read_csv(in_path+f ).toxic for f in files])
 pb = np.array([0.9459,0.9416,0.9422,0.9399,0.9158] )<compute_test_metric>
cities = shop['shop_name'].str.split(' ' ).map(lambda row: row[0] )
weight = lambda x: softmax(1/(1-x))<prepare_output>
shop['city'] = shop['shop_name'].str.split(' ' ).map(lambda row: row[0])
 shop.loc[shop.city == '!Якутск', 'city'] = 'Якутск'
w = weight(pb)
 print(w)
 sub = pd.read_csv(path+'sample_submission.csv')
 sub['toxic'] = sumbs.T@w
sub.head()<feature_engineering>
pr = preprocessing.LabelEncoder()
pr.fit_transform(shop['city'] )
test = pd.read_csv(path+'test.csv')
 dic_ids = {k:v.id for k,v in test.groupby(["lang"])}
adj = {
"fr":1.04,
"es":1.06,
"pt":.96,
"it":.97,
"tr":.98,
}
for l,v in adj.items() :
ids = dic_ids[l]
sub.loc[ids,"toxic"] *= v
sub.head()<save_to_csv>
shop['cities_label'] = pr.fit_transform(shop['city'])
 shop.drop(['shop_name', 'city'], axis = 1, inplace=True )
sub.to_csv('submission.csv', index=False )<save_to_csv>
item_train = train['item_id'].nunique()
item_test = test['item_id'].nunique()
sub.to_csv('submission.csv', index=False )<import_modules>
len(set(item_test_list ).difference(item_train_list))
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer, SentencePieceBPETokenizer<set_options>
items.loc[~items['item_category_id'].isin(items_in_test)].T
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
le = preprocessing.LabelEncoder()
main_items = item_cat['item_category_name'].str.split('-')
 item_cat['main_category_id'] = main_items.map(lambda row: row[0].strip())
 item_cat['main_category_id'] = le.fit_transform(item_cat['main_category_id'] )
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 128<load_pretrained>
item_cat['sub_category_id'] = main_items.map(lambda row: row[1].strip() if len(row)> 1 else row[0].strip())
 item_cat['sub_category_id'] = le.fit_transform(item_cat['sub_category_id'] )
tokenizer = transformers.AutoTokenizer.from_pretrained("xlm-roberta-large")
 tokenizer.save_pretrained('.')
<load_from_csv>
train['date'] = pd.to_datetime(train['date'], format='%d.%m.%Y' )
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
<count_values>
from itertools import product
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=0)
 ])
 print(train.toxic.value_counts())
 print(valid.toxic.value_counts() )<load_from_csv>
jan = list(product(*[shops_in_jan, items_in_jan, [0]]))
olid = pd.read_csv('/kaggle/input/olid2019/olid-training-v1.0.tsv', sep='\t')
 olid = olid.rename(columns={"id": "id", "tweet": "comment_text", 'subtask_a': 'toxic'})
 olid.toxic =(olid.toxic == 'OFF' ).astype(int)
 olid.head()<load_from_csv>
cartesian_test = []
cartesian_test.append(np.array(jan))
cartesian_test.append(np.array(feb))
olid_test = pd.read_csv('/kaggle/input/olid2019/testset-levela.tsv', sep='\t')
 olid_test_label = pd.read_csv('/kaggle/input/olid2019/labels-levela.csv', names = ['id', 'label'] )<count_values>
cartesian_test = np.vstack(cartesian_test )
olid_test_label.label.value_counts()<merge>
cartesian_test_df = pd.DataFrame(cartesian_test, columns = ['shop_id', 'item_id', 'date_block_num'] )
olid_test = olid_test.set_index('id' ).join(olid_test_label.set_index('id'))<data_type_conversions>
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float16)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
olid_test['toxic'] =(olid_test.label == 'OFF' ).astype(int ).values<split>
cartesian_df = pd.DataFrame(np.vstack(cartesian), columns = ['shop_id', 'item_id', 'date_block_num'], dtype=np.int32 )
olid_train, olid_valid = train_test_split(olid, test_size=3240, random_state = 2020)
 olid_train_1k = olid_train[0:1000]
olid_train_2k = olid_train[0:2000]
olid_train_5k = olid_train[0:5000] 
<categorify>
x = train.groupby(['shop_id', 'item_id', 'date_block_num'])['item_cnt_day'].sum().rename('item_cnt_month' ).reset_index()
def fast_encode_xlm(texts, tokenizer, chunk_size=256, maxlen=512):

 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.batch_encode_plus(text_chunk, pad_to_max_length = True, max_length = maxlen)
 all_ids.extend(np.array(encs.input_ids))
return np.array(all_ids )<prepare_x_and_y>
new_train = pd.merge(cartesian_df, x, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0 )
x_train = fast_encode_xlm(train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode_xlm(valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode_xlm(test.content.astype(str), tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<categorify>
new_train['item_cnt_month'] = np.clip(new_train['item_cnt_month'], 0, 20 )
olid_train_encode = fast_encode_xlm(olid_train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_1k_encode = fast_encode_xlm(olid_train_1k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_2k_encode = fast_encode_xlm(olid_train_2k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_5k_encode = fast_encode_xlm(olid_train_5k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_valid_encode = fast_encode_xlm(olid_valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_test_encode = fast_encode_xlm(olid_test.tweet.astype(str), tokenizer, maxlen = MAX_LEN)
<prepare_x_and_y>
new_train.sort_values(['date_block_num','shop_id','item_id'], inplace = True)
 new_train.head()
y_olid_train = olid_train.toxic.values
y_olid_train_1k = olid_train_1k.toxic.values
y_olid_train_2k = olid_train_2k.toxic.values
y_olid_train_5k = olid_train_5k.toxic.values
y_olid_valid = olid_valid.toxic.values
y_olid_test =(olid_test_label.label == 'OFF' ).astype(int ).values<define_variables>
test.insert(loc=3, column='date_block_num', value=34 )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<categorify>
test['item_cnt_month'] = 0
olid_test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(olid_test_encode)
.batch(BATCH_SIZE)
)
 olid_valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_valid_encode, y_olid_valid)) 
.batch(BATCH_SIZE)
)
 olid_train_1k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_1k_encode, y_olid_train_1k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_2k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_2k_encode, y_olid_train_2k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_5k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_5k_encode, y_olid_train_5k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_encode, y_olid_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
<choose_model_class>
new_train = new_train.append(test.drop(['ID'], axis=1))
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_pretrained>
new_train = pd.merge(new_train, shop, on=['shop_id'], how='left')
 new_train.head()
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFAutoModelWithLMHead.from_pretrained('jplu/tf-xlm-roberta-large')
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary() 
<train_model>
new_train = pd.merge(new_train, items.drop('item_name', axis = 1), on=['item_id'], how='left')
 new_train.head()
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
new_train = pd.merge(new_train, item_cat.drop('item_category_name', axis = 1), on=['item_category_id'], how='left')
 new_train.head()
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
def generate_lag(train, months, lag_column):
for month in months:
train_shift = train[['date_block_num', 'shop_id', 'item_id', lag_column]].copy()
train_shift.columns = ['date_block_num', 'shop_id', 'item_id', lag_column+'_lag_'+ str(month)]
train_shift['date_block_num'] += month
train = pd.merge(train, train_shift, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 return train
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False)
<import_modules>
new_train = downcast_dtypes(new_train )
from sklearn.metrics import roc_auc_score<save_to_csv>
gc.collect()
olid_test.toxic_predict = model.predict(olid_test_dataset, verbose=1)
 olid_test.to_csv('olid_test_0shot.csv', index=False)
 roc_auc_score(y_true = olid_test.toxic, y_score = olid_test.toxic_predict )<compute_train_metric>
%%time
new_train = generate_lag(new_train, [1, 2, 3, 4, 5, 6, 12], 'item_cnt_month' )
def Find_Optimal_Cutoff(target, predicted):

 fpr, tpr, threshold = roc_curve(target, predicted)
 i = np.arange(len(tpr))
roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})
 roc_t = roc.iloc[(roc.tf-0 ).abs().argsort() [:1]]
return list(roc_t['threshold'])
 threshold = Find_Optimal_Cutoff(olid_test.toxic, olid_test.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test.toxic_predict]<compute_test_metric>
%%time
group = new_train.groupby(['date_block_num', 'item_id'])['item_cnt_month'].mean().rename('item_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'item_id'], how='left')
 new_train = generate_lag(new_train, [1,2,3,4,5,6,12], 'item_month_mean')
 new_train.drop(['item_month_mean'], axis=1, inplace=True )
f1_score(y_true = olid_test.toxic, y_pred = olid_test.toxic_predict_binary)
<save_to_csv>
%%time
group = new_train.groupby(['date_block_num', 'shop_id'])['item_cnt_month'].mean().rename('shop_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'shop_id'], how='left')
 new_train = generate_lag(new_train, [1,2,3,6,12], 'shop_month_mean')
 new_train.drop(['shop_month_mean'], axis=1, inplace=True )
with strategy.scope() :
model_1k = model
model_1k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_1k.shape[0] // BATCH_SIZE
train_history_1k = model_1k.fit(
 olid_train_1k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test1k = olid_test
olid_test1k.toxic_predict = model_1k.predict(olid_test_dataset, verbose=1)
 olid_test1k.to_csv('olid_test_1k.csv', index=False)
 print('1k roc is ' + str(roc_auc_score(y_true = olid_test1k.toxic, y_score = olid_test1k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test1k.toxic, olid_test1k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test1k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test1k.toxic_predict]
print('1k f1-score is ' + str(f1_score(y_true = olid_test1k.toxic, y_pred = olid_test1k.toxic_predict_binary)))
 del model_1k<save_to_csv>
%%time
group = new_train.groupby(['date_block_num', 'shop_id', 'item_category_id'])['item_cnt_month'].mean().rename('item_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 new_train = generate_lag(new_train, [1, 2], 'item_category_month_mean')
 new_train.drop(['item_category_month_mean'], axis=1, inplace=True )
with strategy.scope() :
model_2k = model
model_2k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_2k.shape[0] // BATCH_SIZE
train_history_2k = model_2k.fit(
 olid_train_2k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test2k = olid_test
olid_test2k.toxic_predict = model_2k.predict(olid_test_dataset, verbose=1)
 olid_test2k.to_csv('olid_test_2k.csv', index=False)
 print('2k roc is ' + str(roc_auc_score(y_true = olid_test2k.toxic, y_score = olid_test2k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test2k.toxic, olid_test2k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test2k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test2k.toxic_predict]
print('2k f1-score is ' + str(f1_score(y_true = olid_test2k.toxic, y_pred = olid_test2k.toxic_predict_binary)))
 del model_2k<save_to_csv>
%%time
group = new_train.groupby(['date_block_num', 'main_category_id'])['item_cnt_month'].mean().rename('main_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'main_category_id'], how='left')
 new_train = generate_lag(new_train, [1], 'main_category_month_mean')
 new_train.drop(['main_category_month_mean'], axis=1, inplace=True )
model_5k = model
model_5k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_5k.shape[0] // BATCH_SIZE
train_history_5 = model_5k.fit(
 olid_train_5k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test5k = olid_test
olid_test5k.toxic_predict = model_5k.predict(olid_test_dataset, verbose=1)
 olid_test5k.to_csv('olid_test_5k.csv', index=False)
 print('5k roc is ' + str(roc_auc_score(y_true = olid_test5k.toxic, y_score = olid_test5k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test5k.toxic, olid_test5k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test5k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test5k.toxic_predict]
print('5k f1-score is ' + str(f1_score(y_true = olid_test5k.toxic, y_pred = olid_test5k.toxic_predict_binary)))
 del model_5k<save_to_csv>
%%time
group = new_train.groupby(['date_block_num', 'sub_category_id'])['item_cnt_month'].mean().rename('sub_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'sub_category_id'], how='left')
 new_train = generate_lag(new_train, [1], 'sub_category_month_mean')
 new_train.drop(['sub_category_month_mean'], axis=1, inplace=True )
model_10k = model
model_10k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train.shape[0] // BATCH_SIZE
train_history_10 = model_10k.fit(
 olid_train_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=20 
)
 olid_test10k = olid_test
olid_test10k.toxic_predict = model_10k.predict(olid_test_dataset, verbose=1)
 olid_test10k.to_csv('olid_test_10k.csv', index=False)
 print('10k roc is ' + str(roc_auc_score(y_true = olid_test10k.toxic, y_score = olid_test10k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test10k.toxic, olid_test10k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test10k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test10k.toxic_predict]
print('10k f1-score is ' + str(f1_score(y_true = olid_test10k.toxic, y_pred = olid_test10k.toxic_predict_binary)))
 del model_10k<import_modules>
new_train = downcast_dtypes(new_train )
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer, SentencePieceBPETokenizer<categorify>
import xgboost as xgb
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<set_options>
new_train = new_train[new_train.date_block_num > 11]
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
gc.collect()
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 128<load_pretrained>
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
df[col].fillna(0, inplace=True)
 return df
tokenizer = transformers.AutoTokenizer.from_pretrained("jplu/tf-xlm-roberta-large")
 tokenizer.save_pretrained('.')
<load_from_csv>
new_train = fill_na(new_train)
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
<load_from_csv>
def xgtrain() :
regressor = xgb.XGBRegressor(n_estimators = 5000,
learning_rate = 0.01,
max_depth = 10,
subsample = 0.5,
colsample_bytree = 0.5)
 regressor_ = regressor.fit(new_train[new_train.date_block_num < 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num < 33]['item_cnt_month'].values,
eval_metric = 'rmse',
eval_set = [(new_train[new_train.date_block_num < 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num < 33]['item_cnt_month'].values), 
(new_train[new_train.date_block_num == 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num == 33]['item_cnt_month'].values)
 ],
verbose=True,
early_stopping_rounds = 20 
)
 return regressor_
olid = pd.read_csv('/kaggle/input/olid2019/olid-training-v1.0.tsv', sep='\t')
 olid = olid.rename(columns={"id": "id", "tweet": "comment_text", 'subtask_a': 'toxic'})
 olid.toxic =(olid.toxic == 'OFF' ).astype(int)
 olid['comment_text'] = olid['comment_text'].str.replace('@USER', '')
 olid.head()<split>
%%time
regressor_ = xgtrain()
olid_train, olid_test = train_test_split(olid, test_size=3240, random_state = 2020)
 olid_train_1k = olid_train[0:1000]
olid_train_2k = olid_train[0:2000]
olid_train_5k = olid_train[0:5000] 
<concatenate>
predictions = regressor_.predict(new_train[new_train.date_block_num == 34].drop(['item_cnt_month'], axis = 1 ).values )
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=0)
 ] )<count_values>
submission = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/sample_submission.csv' )
train.toxic.value_counts()<categorify>
submission['item_cnt_month'] = predictions
def fast_encode_xlm(texts, tokenizer, chunk_size=256, maxlen=512):

 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.batch_encode_plus(text_chunk, pad_to_max_length = True, max_length = maxlen)
 all_ids.extend(np.array(encs.input_ids))
return np.array(all_ids )<prepare_x_and_y>
submission.to_csv('saleslearn.csv', index=False )