Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<load_from_csv>
os.environ['KMP_DUPLICATE_LIB_OK']='True'
curdoc().theme = 'light_minimal'
output_notebook()
PATH = "/kaggle/input/competitive-data-science-predict-future-sales/"
for i_fold in range(n_fold):
if os.path.isfile(SAVE_PATH + f"history_{version_name}_fold_{i_fold}.csv"):
history_file = SAVE_PATH + f"history_{version_name}_fold_{i_fold}.csv"
elif os.path.isfile(MODEL_PATH + f"history_{version_name}_fold_{i_fold}.csv"):
history_file = MODEL_PATH + f"history_{version_name}_fold_{i_fold}.csv"
else:
continue
print(f"show {history_file}")
 history = pd.read_csv(history_file)
 visualize_training_history(history, start_loc = 2 )<load_pretrained>
items_df = pd.read_csv(f"{PATH}items.csv")
 shops_df = pd.read_csv(f"{PATH}shops.csv")
 sales_train_df = pd.read_csv(f"{PATH}sales_train.csv")
 test_df = pd.read_csv(f"{PATH}test.csv")
 sample_submission_df = pd.read_csv(f"{PATH}sample_submission.csv")
 item_categories_df = pd.read_csv(f"{PATH}item_categories.csv" )
def get_models(model_files, model_class):
device = DEVICE if not USE_TPU else xm.xla_device()
models = []
params = []
for model_f in model_files:
if os.path.isfile(SAVE_PATH + model_f):
resume_file_path = SAVE_PATH + model_f
elif os.path.isfile(MODEL_PATH + model_f):
resume_file_path = MODEL_PATH + model_f
else:
continue
model = model_class()
model, _, _, other_params = resume_model(model, resume_file_path = resume_file_path, device = 'cpu')
 model = model.to(device)
 model.eval()
other_params['sub_mode'] = ''
if 'best.' in model_f:
other_params['sub_mode'] = '_best'
elif 'best_score.' in model_f:
other_params['sub_mode'] = '_best_score'
models.append(model)
 params.append(other_params)
 return models, params
@show_running_time('test')
 def predict(model, loader):
device = DEVICE if not USE_TPU else xm.xla_device()
model.eval()
preds, metric_preds = [], []
with torch.no_grad() :
for img_batch in loader:
img_batch = img_batch.to(device)
 logits, metric_logits = model(img_batch)
 preds.append(F.softmax(logits, dim = 1 ).cpu().numpy())
 metric_preds.append(metric_logits.cpu().numpy())
 preds = np.concatenate(preds)
 metric_preds = np.concatenate(metric_preds)
 return preds, metric_preds<predict_on_test>
sales_train_df.loc[sales_train_df.shop_id == 0, 'shop_id'] = 57
test_df.loc[test_df.shop_id == 0, 'shop_id'] = 57
sales_train_df.loc[sales_train_df.shop_id == 1, 'shop_id'] = 58
test_df.loc[test_df.shop_id == 1, 'shop_id'] = 58
sales_train_df.loc[sales_train_df.shop_id == 10, 'shop_id'] = 11
test_df.loc[test_df.shop_id == 10, 'shop_id'] = 11
@clear_cache
def predict_and_submission(test_loader, model_files):
submission1 = submission.copy()
submission2 = submission.copy()
n_fold_preds, n_fold_metric_preds = [], []
models, params = get_models(model_files, model_class)
 for model in models:
preds, metric_preds = predict(model, test_loader)
 n_fold_preds.append(preds)
 n_fold_metric_preds.append(metric_preds)
 sub_name = '_'.join([f"fold_{p['i_fold']}_{p['last_epoch']+1}ep{p['sub_mode']}" for p in params])
 submission_name1 = f"{version_name}_{network_name}_{len(model_files)}_fold_submission1_{sub_name}.csv"
submission_name2 = f"{version_name}_{network_name}_{len(model_files)}_fold_submission2_{sub_name}.csv"
with open(SAVE_PATH + f'log_{version_name}_{network_name}_submission.txt', 'a')as appender:
content =(f"
{time.ctime() }, submission file: {submission_name1}
" +
f"test_data: {IMAGE_FILE_NAME[0]}
test_img_size: {INPUT_IMG_SHAPE}
from models:
")
 for s, p in zip(model_files, params):
content += s + f", i_fold: {p['i_fold']}, train_epochs: {p['last_epoch']+1}ep, best_val_score: {p['best_val_score']:.4f}, best_val_loss: {p['best_val_loss']:.4f}
"
print(content)
 appender.write(content + '
')
 preds = np.stack(n_fold_preds ).mean(axis = 0)
 submission1.iloc[:, 1:] = preds
submission1.to_csv(SAVE_PATH + submission_name1, index=False)
 display(submission1.tail(5))<load_from_csv>
item_to_category = dict(zip(items_df.item_id, items_df.item_category_id))
sales_train_df["item_category_id"] = sales_train_df["item_id"].apply(lambda x: item_to_category[x])
 sales_train_df.head()
df_test = pd.read_csv(PATH + 'test.csv')
 submission = pd.read_csv(PATH + 'sample_submission.csv')
 test_loader = get_test_dataloader()
model_files = [
f"{version_name}_{network_name}_fold_{0}_best.pth",
]
predict_and_submission(test_loader, model_files)
 model_files = [
f"{version_name}_{network_name}_fold_{0}_best_score.pth",
] 
<install_modules>
sales_train_df["date"] = pd.to_datetime(sales_train_df["date"], format="%d.%m.%Y")
 print(f"Training date ranges from: {sales_train_df.date.min() }, to: {sales_train_df.date.max() }" )
!pip install -q efficientnet<import_modules>
sales_aggr = sales_train_df.groupby(["date"])["item_cnt_day"]
sales_aggr_mean = sales_aggr.agg("mean")
 sales_aggr_sum = sales_aggr.agg("sum" )
import numpy as np
import pandas as pd
import os
import tensorflow as tf
import keras
import efficientnet.tfkeras as efn
from kaggle_datasets import KaggleDatasets
from tensorflow.keras.models import Sequential
import tensorflow.keras.layers as L
from tensorflow.keras.applications import InceptionResNetV2
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt<set_options>
sales_aggr_month = sales_train_df[["date", "item_cnt_day"]]
sales_aggr_month["date"] = sales_train_df.date.dt.month
sales_aggr_month = sales_aggr_month.groupby(["date"])["item_cnt_day"]
sales_aggr_month_mean = sales_aggr_month.agg("mean")
 sales_aggr_month_sum = sales_aggr_month.agg("sum" )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
def seed_everything(seed=0):
np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 2048
seed_everything(seed)
 print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 40
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<define_variables>
item_aggr = items_df[["item_id", "item_category_id"]].groupby(["item_category_id"])
 item_aggr_cnt = item_aggr.count()
item_aggr_cnt.head()
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'<load_from_csv>
shop_aggr = sales_train_df[["item_id", "shop_id"]].groupby(["shop_id"])
 shop_aggr_cnt = shop_aggr.count()
shop_aggr_cnt.head()
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
SPLIT_VALIDATION =True
if SPLIT_VALIDATION:
train_paths, valid_paths, train_labels, valid_labels =train_test_split(train_paths, train_labels, test_size=0.15, random_state=seed)
<define_variables>
neg_cnt = sales_train_df[sales_train_df.item_cnt_day < 0]
neg_shop_aggr = neg_cnt[["item_id", "shop_id"]].groupby(["shop_id"])
 neg_shop_aggr_cnt = neg_shop_aggr.count()
neg_shop_aggr_cnt = neg_shop_aggr_cnt.reindex(range(shop_aggr_cnt.shape[0]), fill_value=0)
 neg_shop_aggr_cnt.head()
image_size = 800<normalization>
print(f"Corr coef between the lines: {np.corrcoef(shop_norm, neg_shop_norm)[0, 1]:.4f}" )
def decode_image(filename, label=None, image_size=(image_size, image_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label<categorify>
test_df = reduce_mem_usage(test_df)
 items_df = reduce_mem_usage(items_df)
 shops_df = reduce_mem_usage(shops_df)
 item_categories_df = reduce_mem_usage(item_categories_df )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 train_dataset_1 =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(64)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<init_hyperparams>
previous_count = sales_train_df.shape[0]
sales_train_df = sales_train_df[sales_train_df.item_price < unique_prices[-1]]
sales_train_df = sales_train_df[sales_train_df.item_cnt_day < unique_count[-1]]
new_count = sales_train_df.shape[0]
print(f"Reduced: {previous_count - new_count} values" )
LR_START = 0.0001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.0001
LR_RAMPUP_EPOCHS = 4
LR_SUSTAIN_EPOCHS = 6
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<train_model>
sales_train_df.loc[sales_train_df.item_price < 0, "item_price"] = np.median(sales_train_df.loc[(sales_train_df.date_block_num==4)
 &(sales_train_df.shop_id==32)
 &(sales_train_df.item_id==2973),
"item_price"] )
<choose_model_class>
shops_df.loc[shops_df.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops_df['city'] = shops_df['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops_df.loc[shops_df.city == '!Якутск', 'city'] = 'Якутск'
shops_df['city_code'] = LabelEncoder().fit_transform(shops_df['city'])
 shops_df = shops_df[['shop_id','city_code']]
item_categories_df['split'] = item_categories_df['item_category_name'].str.split('-')
 item_categories_df['type'] = item_categories_df['split'].map(lambda x: x[0].strip())
 item_categories_df['type_code'] = LabelEncoder().fit_transform(item_categories_df['type'])
 item_categories_df['subtype'] = item_categories_df['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 item_categories_df['subtype_code'] = LabelEncoder().fit_transform(item_categories_df['subtype'])
 item_categories_df = item_categories_df[['item_category_id','type_code', 'subtype_code']]
items_df.drop(['item_name'], axis=1, inplace=True )
with strategy.scope() :
model = tf.keras.Sequential([
InceptionResNetV2(
 input_shape=(image_size, image_size, 3),
weights='imagenet',
include_top=False 
),
L.GlobalMaxPooling2D() ,
L.Dense(4, activation='softmax')
 ])
 model.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model.summary()<train_model>
sales_train_df['revenue'] = sales_train_df['item_price'] * sales_train_df['item_cnt_day']
STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_callback],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset if SPLIT_VALIDATION else None,
)<choose_model_class>
full_data = reduce_mem_usage(full_data )
with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(image_size, image_size, 3),
weights='noisy-student',
include_top=False 
),
L.GlobalMaxPooling2D() ,
L.Dense(4, activation='softmax')
 ])
 model2.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model2.summary()<train_model>
group = sales_train_df.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 full_data = pd.merge(full_data, group, on=['date_block_num','shop_id','item_id'], how='left')
 full_data['item_cnt_month'] =(full_data['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
STEPS_PER_EPOCH = train_labels.shape[0] // 64
history = model2.fit(
 train_dataset_1,
epochs=EPOCHS,
callbacks=[lr_callback],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset if SPLIT_VALIDATION else None,
)<save_to_csv>
test_df['date_block_num'] = 34
test_df['date_block_num'] = test_df['date_block_num'].astype(np.int8)
 test_df['shop_id'] = test_df['shop_id'].astype(np.int8)
 test_df['item_id'] = test_df['item_id'].astype(np.int16 )
probs1 = model.predict(test_dataset, verbose=1)
 probs2 = model2.predict(test_dataset, verbose=1)
 probs_avg =(probs1+probs2)/2
sub.loc[:, 'healthy':] = probs_avg
sub.to_csv('submission.csv', index=False)
 sub.head()<install_modules>
full_data = pd.concat([full_data, test_df], ignore_index=True, sort=False, keys=['date_block_num','shop_id','item_id'])
 full_data.fillna(0, inplace=True )
!pip install -q efficientnet<set_options>
full_data = pd.merge(full_data, shops_df, on=['shop_id'], how='left')
 full_data = pd.merge(full_data, items_df, on=['item_id'], how='left')
 full_data = pd.merge(full_data, item_categories_df, on=['item_category_id'], how='left')
 full_data['city_code'] = full_data['city_code'].astype(np.int8)
 full_data['item_category_id'] = full_data['item_category_id'].astype(np.int8)
 full_data['type_code'] = full_data['type_code'].astype(np.int8)
 full_data['subtype_code'] = full_data['subtype_code'].astype(np.int8 )
tqdm.pandas()
np.random.seed(0)
 tf.random.set_seed(0)
 warnings.filterwarnings("ignore" )<load_from_csv>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
EPOCHS = 20
SAMPLE_LEN = 100
IMAGE_PATH = ".. /input/plant-pathology-2020-fgvc7/images/"
TEST_PATH = ".. /input/plant-pathology-2020-fgvc7/test.csv"
TRAIN_PATH = ".. /input/plant-pathology-2020-fgvc7/train.csv"
SUB_PATH = ".. /input/plant-pathology-2020-fgvc7/sample_submission.csv"
sub = pd.read_csv(SUB_PATH)
 test_data = pd.read_csv(TEST_PATH)
 train_data = pd.read_csv(TRAIN_PATH )<define_variables>
%%time
date_item_group = full_data.groupby(["date_block_num", "item_id"] ).agg({"item_cnt_month": ["mean"]})
 date_item_group.columns = ["date_item_avg_item_cnt"]
date_item_group.reset_index(inplace=True)
 full_data = pd.merge(full_data, date_item_group, on=["date_block_num", "item_id"], how="left")
 full_data = lag_feature(full_data, [1, 2, 3, 6, 12], "date_item_avg_item_cnt")
 full_data.drop(["date_item_avg_item_cnt"], axis=1, inplace=True)
 date_item_group = full_data.groupby(["date_block_num", "shop_id"] ).agg({"item_cnt_month": ["mean"]})
 date_item_group.columns = ["date_shop_avg_item_cnt"]
date_item_group.reset_index(inplace=True)
 full_data = pd.merge(full_data, date_item_group, on=["date_block_num", "shop_id"], how="left")
 full_data = lag_feature(full_data, [1, 2, 3, 6, 12], "date_shop_avg_item_cnt")
 full_data.drop(["date_shop_avg_item_cnt"], axis=1, inplace=True)
 date_item_group = full_data.groupby(["date_block_num", "item_id", "shop_id"] ).agg({"item_cnt_month": ["mean"]})
 date_item_group.columns = ["date_item_shop_avg_item_cnt"]
date_item_group.reset_index(inplace=True)
 full_data = pd.merge(full_data, date_item_group, on=["date_block_num", "item_id", "shop_id"], how="left")
 full_data = lag_feature(full_data, [1, 2, 3], "date_item_shop_avg_item_cnt")
 full_data.drop(["date_item_shop_avg_item_cnt"], axis=1, inplace=True)
 date_shop_cat_group = full_data.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 date_shop_cat_group.columns = ['date_shop_cat_avg_item_cnt']
date_shop_cat_group.reset_index(inplace=True)
 full_data = pd.merge(full_data, date_shop_cat_group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 full_data = lag_feature(full_data, [1], 'date_shop_cat_avg_item_cnt')
 full_data.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 item_group = sales_train_df.groupby(["item_id"] ).agg({"item_price": ["mean"]})
 item_group.columns = ["item_avg_item_price"]
item_group.reset_index(inplace=True)
 full_data = pd.merge(full_data, item_group, on=["item_id"], how="left")
 date_item_group = sales_train_df.groupby(["date_block_num", "item_id"] ).agg({"item_price": ["mean"]})
 date_item_group.columns = ["date_item_avg_item_price"]
date_item_group.reset_index(inplace=True)
 full_data = pd.merge(full_data, date_item_group, on=["date_block_num", "item_id"], how="left")
 lags = [1,2,3,4,5,6]
full_data = lag_feature(full_data, lags, 'date_item_avg_item_price')
 for i in lags:
full_data['delta_price_lag_'+str(i)] = \ 
(full_data['date_item_avg_item_price_lag_'+str(i)] - full_data['item_avg_item_price'])/ full_data['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
full_data['delta_price_lag'] = full_data.apply(select_trend, axis=1)
 full_data['delta_price_lag'] = full_data['delta_price_lag'].astype(np.float16)
 full_data['delta_price_lag'].fillna(0, inplace=True)
red_values = [np.mean(train_images[idx][:, :, 0])for idx in range(len(train_images)) ]
green_values = [np.mean(train_images[idx][:, :, 1])for idx in range(len(train_images)) ]
blue_values = [np.mean(train_images[idx][:, :, 2])for idx in range(len(train_images)) ]
values = [np.mean(train_images[idx])for idx in range(len(train_images)) ]<feature_engineering>
group = sales_train_df.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 full_data = pd.merge(full_data, group, on=['date_block_num','shop_id'], how='left')
 full_data['date_shop_revenue'] = full_data['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 full_data = pd.merge(full_data, group, on=['shop_id'], how='left')
 full_data['shop_avg_revenue'] = full_data['shop_avg_revenue'].astype(np.float32)
 full_data['delta_revenue'] =(full_data['date_shop_revenue'] - full_data['shop_avg_revenue'])/ full_data['shop_avg_revenue']
full_data['delta_revenue'] = full_data['delta_revenue'].astype(np.float16)
 full_data = lag_feature(full_data, [1], 'delta_revenue')
 full_data.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True )
invert(train_images[3])
 invert(train_images[4])
 invert(train_images[5] )<choose_model_class>
full_data['month'] = full_data['date_block_num'] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 full_data['days'] = full_data['month'].map(days ).astype(np.int8 )
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_DS_PATH = KaggleDatasets().get_gcs_path()<split>
full_data = full_data[full_data.date_block_num > 11]
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
test_paths = test_data.image_id.apply(format_path ).values
train_paths = train_data.image_id.apply(format_path ).values
train_labels = np.float32(train_data.loc[:, 'healthy':'scab'].values)
 train_paths, valid_paths, train_labels, valid_labels =\
train_test_split(train_paths, train_labels, test_size=0.15, random_state=2020 )<normalization>
%%time
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
full_data = fill_na(full_data )
def decode_image(filename, label=None, image_size=(512, 512)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label<categorify>
full_data.drop(["ID"], axis=1, inplace=True )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<choose_model_class>
cat_col = ["date_block_num", "shop_id", "item_id", "item_category_id", "month", "days"]
full_data[cat_col] = full_data[cat_col].astype(np.int32 )
def build_lrfn(lr_start=0.00001, lr_max=0.00005,
lr_min=0.00001, lr_rampup_epochs=5,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)*\
lr_exp_decay**(epoch - lr_rampup_epochs\
- lr_sustain_epochs)+ lr_min
return lr
return lrfn<choose_model_class>
X_train = full_data[full_data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 y_train = full_data[full_data.date_block_num < 33]['item_cnt_month']
X_valid = full_data[full_data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 y_valid = full_data[full_data.date_block_num == 33]['item_cnt_month']
X_test = full_data[full_data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
with strategy.scope() :
model = tf.keras.Sequential([DenseNet121(input_shape=(512, 512, 3),
weights='imagenet',
include_top=False),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1],
activation='softmax')])
 model.compile(optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'])
 model.summary()<train_model>
X_train.drop(["delta_price_lag_1", "delta_price_lag_2", "delta_price_lag_3", "delta_price_lag_4", "delta_price_lag_5", "delta_price_lag_6"], axis=1, inplace=True)
 X_valid.drop(["delta_price_lag_1", "delta_price_lag_2", "delta_price_lag_3", "delta_price_lag_4", "delta_price_lag_5", "delta_price_lag_6"], axis=1, inplace=True)
 X_test.drop(["delta_price_lag_1", "delta_price_lag_2", "delta_price_lag_3", "delta_price_lag_4", "delta_price_lag_5", "delta_price_lag_6"], axis=1, inplace=True )
history = model.fit(train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset )<predict_on_test>
cat_feats = [i for i, c in enumerate(X_train.columns)if c in cat_col]
def process(img):
return cv2.resize(img/255.0,(512, 512)).reshape(-1, 512, 512, 3)
 def predict(img):
return model.layers[2](model.layers[1](model.layers[0](process(img)))).numpy() [0]
fig = make_subplots(rows=4, cols=2)
 preds = predict(train_images[2])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Scab"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Multiple diseases"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Healthy"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[2],(205, 136))), row=1, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=1, col=2)
 fig.update_layout(height=1200, width=800, title_text="DenseNet Predictions", showlegend=False)
 preds = predict(train_images[0])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Multiple diseases"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[0],(205, 136))), row=2, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=2, col=2)
 preds = predict(train_images[3])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Rust"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[3],(205, 136))), row=3, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=3, col=2)
 preds = predict(train_images[1])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Scab"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[1],(205, 136))), row=4, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=4, col=2)
 fig.update_layout(template="plotly_white" )<save_to_csv>
del full_data
gc.collect()
probs_dnn = model.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs_dnn
sub.to_csv('submission_dnn.csv', index=False)
 sub.head()<choose_model_class>
X_train.to_csv("X_train.csv", index=False)
 X_valid.to_csv("X_valid.csv", index=False)
 X_test.to_csv("X_test.csv", index=False)
 y_train.to_csv("y_train.csv", index=False)
 y_valid.to_csv("y_valid.csv", index=False )
with strategy.scope() :
model = tf.keras.Sequential([efn.EfficientNetB7(input_shape=(512, 512, 3),
weights='imagenet',
include_top=False),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1],
activation='softmax')])
 model.compile(optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'])
 model.summary()<train_model>
%%time
xgb_model = XGBRegressor(max_depth=10,
n_estimators=1000,
min_child_weight=1000,
colsample_bytree=0.7,
subsample=0.7,
eta=0.3,
seed=0)
 xgb_model.fit(X_train,
y_train,
eval_metric="rmse",
eval_set=[(X_train, y_train),(X_valid, y_valid)],
verbose=20,
early_stopping_rounds=10)
history = model.fit(train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset )<predict_on_test>
%%time
xgb_train_pred = xgb_model.predict(X_train)
 xgb_val_pred = xgb_model.predict(X_valid)
 xgb_test_pred = xgb_model.predict(X_test)
 print('Train rmse:', np.sqrt(mean_squared_error(y_train, xgb_train_pred)))
 print('Validation rmse:', np.sqrt(mean_squared_error(y_valid, xgb_val_pred)) )
def process(img):
return cv2.resize(img/255.0,(512, 512)).reshape(-1, 512, 512, 3)
 def predict(img):
return model.layers[2](model.layers[1](model.layers[0](process(img)))).numpy() [0]
fig = make_subplots(rows=4, cols=2)
 preds = predict(train_images[2])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Scab"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Multiple diseases"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Healthy"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[2],(205, 136))), row=1, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=1, col=2)
 fig.update_layout(height=1200, width=800, title_text="EfficientNet Predictions", showlegend=False)
 preds = predict(train_images[0])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Multiple diseases"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[0],(205, 136))), row=2, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=2, col=2)
 preds = predict(train_images[3])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Rust"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[3],(205, 136))), row=3, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=3, col=2)
 preds = predict(train_images[1])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Scab"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[1],(205, 136))), row=4, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=4, col=2)
 fig.update_layout(template="plotly_white" )<save_to_csv>
xgb_model.save_model("xgboost")
 del xgb_model, xgb_train_pred
gc.collect()
probs_efn = model.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs_efn
sub.to_csv('submission_efn.csv', index=False)
 sub.head()<choose_model_class>
submission = pd.DataFrame(test_df['ID'], columns=['ID'])
 submission['item_cnt_month'] = xgb_test_pred.clip(0., 20.)
submission.to_csv('submission_xgb.csv', index=False)
 submission.head(10 )
with strategy.scope() :
model = tf.keras.Sequential([efn.EfficientNetB7(input_shape=(512, 512, 3),
weights='noisy-student',
include_top=False),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1],
activation='softmax')])
 model.compile(optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'])
 model.summary()<train_model>
%%time
model=LGBMRegressor(
 n_estimators=200,
learning_rate=0.03,
num_leaves=42,
colsample_bytree=0.95,
subsample=0.8,
max_depth=9,
reg_alpha=0.03,
reg_lambda=0.07,
min_split_gain=0.02,
min_child_weight=40,
silent=False)
 model.fit(X_train, y_train )
history = model.fit(train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset )<predict_on_test>
%%time
lgb_train_pred = model.predict(X_train)
 lgb_val_pred = model.predict(X_valid)
 lgb_test_pred = model.predict(X_test)
 print('Train rmse:', np.sqrt(mean_squared_error(y_train, lgb_train_pred)))
 print('Validation rmse:', np.sqrt(mean_squared_error(y_valid, lgb_val_pred)))
def process(img):
return cv2.resize(img/255.0,(512, 512)).reshape(-1, 512, 512, 3)
 def predict(img):
return model.layers[2](model.layers[1](model.layers[0](process(img)))).numpy() [0]
fig = make_subplots(rows=4, cols=2)
 preds = predict(train_images[2])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Scab"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Multiple diseases"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Healthy"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[2],(205, 136))), row=1, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=1, col=2)
 fig.update_layout(height=1200, width=800, title_text="EfficientNet NoisyStudent Predictions", showlegend=False)
 preds = predict(train_images[0])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Multiple diseases"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[0],(205, 136))), row=2, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=2, col=2)
 preds = predict(train_images[3])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Rust"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[3],(205, 136))), row=3, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=3, col=2)
 preds = predict(train_images[1])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Scab"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[1],(205, 136))), row=4, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=4, col=2)
 fig.update_layout(template="plotly_white" )<save_to_csv>
model.booster_.save_model('lgb_classifier.txt')
 del model, lgb_train_pred
gc.collect()
probs_efnns = model.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs_efnns
sub.to_csv('submission_efnns.csv', index=False)
 sub.head()<save_to_csv>
final_test_preds = 0.5 * lgb_test_pred + 0.5 * xgb_test_pred
final_test_preds = final_test_preds.clip(0., 20.)
ensemble_1, ensemble_2, ensemble_3 = [sub]*3
ensemble_1.loc[:, 'healthy':] = 0.50*probs_dnn + 0.50*probs_efn
ensemble_2.loc[:, 'healthy':] = 0.25*probs_dnn + 0.75*probs_efn
ensemble_3.loc[:, 'healthy':] = 0.75*probs_dnn + 0.25*probs_efn
ensemble_1.to_csv('submission_ensemble_1.csv', index=False)
 ensemble_2.to_csv('submission_ensemble_2.csv', index=False)
 ensemble_3.to_csv('submission_ensemble_3.csv', index=False )<import_modules>
submission = pd.DataFrame(test_df['ID'], columns=['ID'])
 submission['item_cnt_month'] = final_test_preds
submission.to_csv('submission.csv', index=False)
 submission.head(10 )
<install_modules>
val_predictions = pd.DataFrame({"f0": xgb_val_pred,"f1": lgb_val_pred}, index=range(len(xgb_val_pred)))
 test_predictions = pd.DataFrame({"f0": xgb_test_pred,"f1": lgb_test_pred}, index=range(len(xgb_test_pred)) )
!pip install -q efficientnet<import_modules>
final_model = LinearRegression(normalize=True)
 final_model.fit(val_predictions, y_valid )
import math, re, os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
from sklearn import metrics
from sklearn.model_selection import train_test_split
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.vgg19 import VGG19
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2
from tensorflow.keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201
from tensorflow.keras.applications.xception import Xception
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2
from tensorflow.keras.applications.nasnet import NASNetLarge
from efficientnet.tfkeras import EfficientNetB7, EfficientNetL2
from tensorflow.keras.layers import Flatten,Dense,Dropout,BatchNormalization
from tensorflow.keras.models import Model,Sequential
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization
from keras.optimizers import Adam,SGD,Adagrad,Adadelta,RMSprop
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint<choose_model_class>
val_pred_pred = final_model.predict(val_predictions)
 test_pred_pred = final_model.predict(test_predictions )
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
print('Rmse:', np.sqrt(mean_squared_error(y_valid, val_pred_pred)) )
<define_variables><EOS> 
submission = pd.DataFrame(test_df['ID'], columns=['ID'])
 submission['item_cnt_month'] = test_pred_pred.clip(0., 20.)
submission.to_csv('submission_stack.csv', index=False)
 submission.head(10 )