Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<categorify>
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
root_path = '/kaggle/input/competitive-data-science-predict-future-sales/'
df_test = pd.read_csv(root_path + 'test.csv')
 df_item_categories = pd.read_csv(root_path + 'item_categories.csv')
 df_sales_train = pd.read_csv(root_path + 'sales_train.csv')
 df_sample_submission = pd.read_csv(root_path + 'sample_submission.csv')
 df_items = pd.read_csv(root_path + 'items.csv')
 df_shops = pd.read_csv(root_path + 'shops.csv')
 df_categories_translated = pd.read_csv('/kaggle/input/translated/item_categories-translated.csv')
 df_shops_translated = pd.read_csv('/kaggle/input/translated/shops-translated.csv')
 print('The files have been loaded!')
 print('df_test')
 print(df_test.shape)
 display(df_test.head(3))
print('df_item_categories')
 print(df_item_categories.shape)
 display(df_item_categories.head(3))
print('df_sales_train')
 print(df_sales_train.shape)
 display(df_sales_train.head(3))
print('df_sample_submission')
 print(df_sample_submission.shape)
 display(df_sample_submission.head(3))
print('df_items')
 print(df_items.shape)
 display(df_items.head(3))
print('df_shops')
 print(df_shops.shape)
 display(df_shops.head(3))
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<set_options>
print('The number of unique shop ids is', df_test['shop_id'].nunique())
 print('The number of unique item ids is', df_test['item_id'].nunique())
 print('The average number of item ids per shop is', df_test.groupby('shop_id' ).agg(count=('item_id', 'count')).mean() [0] )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
train_combos = df_sales_train.drop_duplicates(['shop_id','item_id'] ).shape[0]
test_combos = df_test.drop_duplicates(['shop_id','item_id'] ).shape[0]
seen_combos = pd.merge(df_sales_train.drop_duplicates(['shop_id','item_id']), df_test, on=['shop_id','item_id'], how='inner' ).shape[0]
print("The number of train combos is {} and the number of test combos is {}".format(train_combos, test_combos))
print("The number of combos in test that are present in train is {}".format(seen_combos))
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
df_test = pd.merge(df_test, df_items[['item_id','item_category_id']], on=['item_id'], how='left')
 df_sales_train = pd.merge(df_sales_train, df_items[['item_id','item_category_id']], on=['item_id'], how='left')
 unseen_item_categories = set(df_test['item_category_id'])- set(df_sales_train['item_category_id'])
 print('The number of item categories that have never been sold before is', len(unseen_item_categories))
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
unseen_items = set(df_test['item_id'])- set(df_sales_train['item_id'])
 print('The number of items that have never been sold before is', len(unseen_items))
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<concatenate>
del df_sales_timeseries, unseen_items, unseen_item_categories, df_item_price, df_item_category_price
gc.collect()
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] )<prepare_x_and_y>
df_sales_train.loc[df_sales_train.shop_id == 0, 'shop_id'] = 57
df_test.loc[df_test.shop_id == 0, 'shop_id'] = 57
df_sales_train.loc[df_sales_train.shop_id == 1, 'shop_id'] = 58
df_test.loc[df_test.shop_id == 1, 'shop_id'] = 58
df_sales_train.loc[df_sales_train.shop_id == 11, 'shop_id'] = 10
df_test.loc[df_test.shop_id == 11, 'shop_id'] = 10
df_sales_train.loc[df_sales_train.shop_id == 40, 'shop_id'] = 39
df_test.loc[df_test.shop_id == 40, 'shop_id'] = 39
df_shops_translated['shop_name'] = df_shops_translated['shop_name_translated'].apply(lambda x: x.lower() ).str.replace('[^\w\s]', '' ).str.replace('\d+','' ).str.strip()
df_shops_translated['shop_city'] = df_shops_translated['shop_name'].str.partition(' ')[0]
display(df_shops_translated[df_shops_translated['shop_id'].isin([0,57,1,58,11,10,40,39])])
 df_shops_translated['city_code'] = LabelEncoder().fit_transform(df_shops_translated['shop_city'])
 df_shops_translated = df_shops_translated[['shop_id','city_code']]
df_categories_translated['split'] = df_categories_translated['item_category_name_translated'].str.lower().str.split('-')
 df_categories_translated['type'] = df_categories_translated['split'].map(lambda x: x[0].strip())
 df_categories_translated['subtype'] = df_categories_translated['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 display(df_categories_translated.head())
 df_categories_translated['type_code'] = LabelEncoder().fit_transform(df_categories_translated['type'])
 df_categories_translated['subtype_code'] = LabelEncoder().fit_transform(df_categories_translated['subtype'])
 df_categories_translated = df_categories_translated[['item_category_id','type_code','subtype_code']]
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<define_variables>
df_prices_shop_item = df_sales_train[['shop_id','item_id','item_price']].drop_duplicates(subset=['shop_id','item_id'], keep='last')
 df_prices_item = df_sales_train[['item_id','item_price']].drop_duplicates(subset=['item_id'], keep='last')
 df_prices_category = df_sales_train.groupby('item_category_id' ).agg(item_price=('item_price','mean')).reset_index()
df_test = df_grid[df_grid['date_block_num']==34]
df_test = pd.merge(df_test.drop('item_price',axis=1), df_prices_shop_item, on=['shop_id','item_id'], how='left')
 item_price = pd.merge(df_test[pd.isnull(df_test['item_price'])], df_prices_item, on='item_id',
how='left' ).set_index(df_test[pd.isnull(df_test['item_price'])].index)['item_price_y']
df_test['item_price'] = df_test['item_price'].fillna(item_price)
 category_price = pd.merge(df_test[pd.isnull(df_test['item_price'])], df_prices_category, on='item_category_id',
how='left' ).set_index(df_test[pd.isnull(df_test['item_price'])].index)['item_price_y']
df_test['item_price'] = df_test['item_price'].fillna(category_price)
 df_test['item_price_bin'] = pd.cut(df_test['item_price'], bins=bins, labels=np.arange(1,8), include_lowest=True)
 df_grid = pd.concat([df_grid[df_grid['date_block_num']<34], df_test[df_grid.columns]], axis=0)
 df_grid['item_price_bin'] = df_grid['item_price_bin'].astype(int)
 del df_prices_shop_item, df_prices_item, df_prices_category, df_test, df_sales_train
gc.collect()
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
df_grid['date_item_mean'] = df_grid.groupby(['date_block_num','item_id'])['item_cnt_month'].transform('mean')
 df_grid['date_category_mean'] = df_grid.groupby(['date_block_num','item_category_id'])['item_cnt_month'].transform('mean')
 df_grid['date_item_shop_mean'] = df_grid.groupby(['date_block_num','item_id','shop_id'])['item_cnt_month'].transform('mean')
 df_grid['date_item_category_mean'] = df_grid.groupby(['date_block_num','item_id','item_category_id'])['item_cnt_month'].transform('mean')
 df_grid['date_shop_category_mean'] = df_grid.groupby(['date_block_num','shop_id','item_category_id'])['item_cnt_month'].transform('mean')
 df_grid['month'] = df_grid['date_block_num'] % 12
number_of_days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 df_grid['days'] = df_grid['month'].map(number_of_days)
 df_grid['months_since_item_sale'] = df_grid['date_block_num'] - df_grid.groupby('item_id')['date_block_num'].transform('min')
 df_grid['months_since_item_shop_sale'] = df_grid['date_block_num'] - df_grid.groupby(['item_id','shop_id'])['date_block_num'].transform('min' )
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
df_grid['shop_item_revenue'] = df_grid['item_cnt_month']*df_grid['item_price']
df_revenue = df_grid.groupby(['shop_id','date_block_num'] ).agg(shop_revenue=('shop_item_revenue',sum)).reset_index()
df_grid = pd.merge(df_grid, df_revenue, on=['shop_id','date_block_num'], how='left')
 df_grid.drop('shop_item_revenue', axis=1, inplace=True)
 del df_revenue
gc.collect()
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
shop_revenue_cols = [col for col in df_grid.columns if 'shop_revenue' in col]
cols_to_downcast = list(set(df_grid.columns)- set(shop_revenue_cols))
df_grid[shop_revenue_cols] = df_grid[shop_revenue_cols].astype(np.float32)
 df_grid[cols_to_downcast] = downcast_dtypes(df_grid[cols_to_downcast] )
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 df.drop(col, axis=1, inplace=True)
 del shifted
return(df)
 mean_encoded = [col for col in df_grid.columns if 'mean' in col]
cols_to_lag = ['shop_revenue','item_price'] + mean_encoded
for col in cols_to_lag:
df_grid = lag_feature(df_grid, [1,2,3,6,12], col)
 print('Lagged values for {} have been calculated'.format(col))
df_grid = df_grid[df_grid['date_block_num']>=12]
lagged_cols = [col for col in df_grid.columns if 'lag' in col]
df_grid[lagged_cols] = df_grid[lagged_cols].fillna(0)
 df_grid.reset_index(drop=True, inplace=True)
 gc.collect()
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<import_modules>
df_grid = pd.read_feather('/kaggle/input/data-prep/df_sales.feather')
 def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float16)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
shop_revenue_cols = [col for col in df_grid.columns if 'shop_revenue' in col]
cols_to_downcast = list(set(df_grid.columns)- set(shop_revenue_cols))
df_grid[shop_revenue_cols] = df_grid[shop_revenue_cols].astype(np.float32)
 df_grid[cols_to_downcast] = downcast_dtypes(df_grid[cols_to_downcast] )
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
train = df_grid[df_grid['date_block_num']<33]
valid = df_grid[df_grid['date_block_num']==33]
X_test = df_grid[df_grid['date_block_num']==34].drop('item_cnt_month', axis=1)
 X_train = train[train['item_cnt_month']>0].drop('item_cnt_month', axis=1)
 y_train = train[train['item_cnt_month']>0]['item_cnt_month']
X_valid = valid[valid['item_cnt_month']>0].drop('item_cnt_month', axis=1)
 y_valid = valid[valid['item_cnt_month']>0]['item_cnt_month']
df_classreg_valid = pd.DataFrame()
df_classreg_test = pd.DataFrame()
df_reg_valid = pd.DataFrame()
df_reg_test = pd.DataFrame()
del df_grid
gc.collect()
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
regressor_params = {
'feature_fraction': 0.75,
'metric': 'rmse',
'nthread': 1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.75,
'learning_rate': 0.03,
'objective': 'mse',
'bagging_seed': 2**7,
'num_leaves': 80,
'bagging_freq':1,
'verbose':0
}
lightgbm_train = lightgbm.Dataset(X_train, label=y_train)
 lightgbm_valid = lightgbm.Dataset(X_valid, label=y_valid)
 regressor = lightgbm.train(regressor_params, lightgbm_train, valid_sets=[lightgbm_train, lightgbm_valid], valid_names=['train','valid'], verbose_eval=10, num_boost_round=500, early_stopping_rounds=10)
 print('LightGBM model for regression has been built!')
 df_classreg_valid['lightgbm'] = regressor.predict(valid.drop('item_cnt_month', axis=1))
df_classreg_test['lightgbm'] = regressor.predict(X_test)
 lightgbm.plot_importance(regressor, figsize=(10,12))
regressor.save_model('lightgbm_classreg_regressor.txt', num_iteration=regressor.best_iteration)
 del lightgbm_train, lightgbm_valid, regressor
gc.collect()
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
cat_features = ['shop_id','item_id','item_category_id','item_price_bin','month','date_block_num','city_code','type_code','subtype_code']
regressor = CatBoostRegressor(
 iterations=100,
random_seed=0,
learning_rate=0.1,
max_ctr_complexity=3,
has_time=True,
boosting_type='Ordered',
loss_function='RMSE',
od_type='Iter',
od_wait=10, 
)
 regressor.fit(
 X_train, y_train,
cat_features=cat_features,
eval_set=(X_valid, y_valid),
verbose=10 
)
 print('CatBoost model for regression has been built!')
 df_classreg_valid['catboost'] = regressor.predict(valid.drop(['item_cnt_month'], axis=1))
df_classreg_test['catboost'] = regressor.predict(X_test)
 print(regressor.get_feature_importance(prettified=True))
regressor.save_model('catboost_classreg_regressor.bin')
 del regressor
gc.collect()
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<set_options>
classifier_params = {
'feature_fraction': 0.75,
'metric': 'auc',
'nthread': 1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.75,
'learning_rate': 0.3,
'objective': 'binary',
'bagging_seed': 2**7,
'num_leaves': 80,
'bagging_freq':1,
'is_unbalance':True,
'verbose':0
}
lightgbm_train = lightgbm.Dataset(train.drop('item_cnt_month', axis=1), label=train['item_cnt_month'].apply(lambda x: 1 if x >= 1 else 0))
lightgbm_valid = lightgbm.Dataset(valid.drop('item_cnt_month', axis=1), label=valid['item_cnt_month'].apply(lambda x: 1 if x >= 1 else 0))
classifier = lightgbm.train(classifier_params, lightgbm_train, valid_sets=[lightgbm_train, lightgbm_valid], valid_names=['train','valid'], verbose_eval=10, num_boost_round=300, early_stopping_rounds=10)
 print('Model for classification has been built!')
 df_classreg_valid['prob'] = classifier.predict(valid.drop('item_cnt_month', axis=1))
df_classreg_test['prob'] = classifier.predict(X_test)
 classifier.save_model('lightgbm_classreg_classifier.txt', num_iteration=classifier.best_iteration)
 del lightgbm_train, lightgbm_valid, classifier
gc.collect()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
df_classreg_valid['average'] =(df_classreg_valid['lightgbm'] + df_classreg_valid['xgboost'] + df_classreg_valid['catboost'])/3
df_classreg_valid['item_cnt_month'] = df_classreg_valid.apply(lambda row: row['average'] if row['prob']>=0.8 else 0, axis=1)
 print('Validation RMSE is', np.sqrt(mean_squared_error(valid['item_cnt_month'], df_classreg_valid['item_cnt_month'])))
 df_classreg_valid.to_csv('classreg_valid.csv', index=False)
 df_classreg_test['average'] =(df_classreg_test['lightgbm'] + df_classreg_test['xgboost']+ df_classreg_test['catboost'])/3
df_classreg_test['item_cnt_month'] = df_classreg_test.apply(lambda row: row['average'] if row['prob']>=0.8 else 0, axis=1)
 df_classreg_test['ID'] = df_sample_submission['ID']
df_classreg_test = df_classreg_test[['ID','item_cnt_month']]
df_classreg_test.to_csv('classreg_test.csv', index=False )
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
path = '/kaggle/input/competitive-data-science-predict-future-sales/sales_train.csv'
train_pairs = pd.read_csv(path)
 train_pairs = train_pairs[train_pairs['date_block_num']<33][['shop_id','item_id']].drop_duplicates(['shop_id','item_id'])
 pairs={(a, b)for a, b in zip(train_pairs.shop_id, train_pairs.item_id)}
items={a for a in train_pairs.item_id}
df_reg_valid['group'] = [2 if(a,b)in pairs else(1 if b in items else 0)for a,b in zip(valid.shop_id, valid.item_id)]
df_reg_test['group'] = [2 if(a,b)in pairs else(1 if b in items else 0)for a,b in zip(X_test.shop_id, X_test.item_id)]
del train_pairs, pairs, items
gc.collect()
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
lightgbm_params = {
'feature_fraction': 0.75,
'metric': 'rmse',
'nthread':1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.75,
'learning_rate': 0.03,
'objective': 'mse',
'bagging_seed': 2**7,
'num_leaves': 80,
'bagging_freq':1,
'verbose':0
}
lightgbm_train = lightgbm.Dataset(train.drop('item_cnt_month', axis=1), label=train['item_cnt_month'])
 lightgbm_valid = lightgbm.Dataset(valid.drop('item_cnt_month', axis=1), label=valid['item_cnt_month'])
 model = lightgbm.train(lightgbm_params, lightgbm_train, valid_sets=[lightgbm_train, lightgbm_valid], valid_names=['train','valid'], verbose_eval=10, num_boost_round=300,
early_stopping_rounds=10)
 df_reg_valid['lightgbm'] = model.predict(valid.drop('item_cnt_month', axis=1))
df_reg_test['lightgbm'] = model.predict(X_test)
 model.save_model('lightgbm_reg.txt', num_iteration=model.best_iteration)
 del lightgbm_train, lightgbm_valid, model
gc.collect()
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<concatenate>
X_train_iloc = train.index[-1] + 1
X_valid_iloc = valid.index[-1] + 1
y_train = train['item_cnt_month']
y_valid = valid['item_cnt_month']
del train, valid
gc.collect()
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] )<prepare_x_and_y>
df_scaled = pd.read_feather('/kaggle/input/scaling/df_scaled.feather')
 df_scaled = downcast_dtypes(df_scaled)
 X_train = df_scaled.iloc[:X_train_iloc]
X_valid = df_scaled.iloc[X_train_iloc:X_valid_iloc]
X_test = df_scaled.iloc[X_valid_iloc:]
del df_scaled
gc.collect()
print('The scaled variables are', X_train.columns )
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<define_variables>
ensembler = XGBRegressor(
 max_depth=2,
n_estimators=150,
min_child_weight=100,
colsample_bytree=0.75,
subsample=0.75,
eta=0.1,
seed=42)
 kfold = KFold(n_splits=5, random_state=42)
 y_pred = cross_val_predict(ensembler, df_reg_valid, y_valid, cv=kfold)
 print('The cross validation RMSE is', np.sqrt(mean_squared_error(y_valid, y_pred)) )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
df_mean_analysis = pd.DataFrame()
df_mean_analysis['group'] = df_reg_valid['group']
df_mean_analysis['lightgbm'] = df_reg_valid['lightgbm']
df_mean_analysis['ridge'] = df_reg_valid['ridge']
df_mean_analysis['ensemble'] = y_pred
df_mean_analysis['target'] = y_valid.values
df_mean_analysis.groupby('group' ).agg(np.mean )
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
df_mean_analysis = pd.DataFrame()
df_mean_analysis['group'] = df_reg_test['group']
df_mean_analysis['lightgbm'] = df_reg_test['lightgbm']
df_mean_analysis['ridge'] = df_reg_test['ridge']
df_mean_analysis['pred'] = ensembler.predict(df_reg_test)
 df_mean_analysis.groupby('group' ).agg(np.mean )
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
del df_mean_analysis
gc.collect()
<save_to_csv><EOS> 
df_reg_valid.to_csv('reg_valid.csv', index=False)
 df_reg_test.to_csv('reg_preds_test.csv', index=False)
 df_reg_test['item_cnt_month'] = ensembler.predict(df_reg_test)
 df_reg_test['ID'] = df_sample_submission['ID']
df_reg_test = df_reg_test[['ID','item_cnt_month']]
df_reg_test.to_csv('reg_test.csv', index=False )