Kaggle competition: Rossmann Store Sales
<SOS> metric: RootMeanSquarePercentageError Kaggle data source: rossmann-store-sales<load_from_csv>
import pandas as pd
from pandas import datetime
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
train_labels = pd.read_csv(base_path/"train.csv")
 path = base_path/"images"<categorify>
df = pd.read_csv('.. /input/rossmann-store-sales/train.csv', parse_dates = ['Date'], low_memory = False)
 df.head()
transform_data(train_labels)
 train_labels = train_labels.set_index("image_id" )<feature_engineering>
df['Date']=pd.to_datetime(df['Date'],format='%Y-%m-%d' )
tfms = get_transforms(max_rotate=30.,flip_vert=True )<categorify>
df['Hour'] = df['Date'].dt.hour
df['Day_of_Month'] = df['Date'].dt.day
df['Day_of_Week'] = df['Date'].dt.dayofweek
df['Month'] = df['Date'].dt.month
src =(ImageList.from_folder(path)
.filter_by_func(lambda fname: "Train" in fname.name)
.split_by_rand_pct() 
.label_from_func(lambda o: train_labels.loc[o.name]['tag']))<categorify>
test = pd.read_csv('.. /input/rossmann-store-sales/test.csv', parse_dates = True, low_memory = False)
 test.head()
def get_data(size, bs):
data =(src.transform(tfms, size=size)
.databunch(bs=bs)
.normalize())
 return data<choose_model_class>
test['Date']=pd.to_datetime(test['Date'],format='%Y-%m-%d' )
def getModelEff(model_name = 'efficientnet-b4'):
model = EfficientNet.from_pretrained(model_name)
 if model_name == 'efficientnet-b4':
model._fc = nn.Linear(1792,4)
 elif model_name == 'efficientnet-b3':
model._fc = nn.Linear(1536,4)
 return model<init_hyperparams>
test['Hour'] = test['Date'].dt.hour
test['Day_of_Month'] = test['Date'].dt.day
test['Day_of_Week'] = test['Date'].dt.dayofweek
test['Month'] = test['Date'].dt.month
archs = {
"resnet50": {
"model": models.resnet50,
"epochs_1": 10,
"epochs_2": 10,
"epochs_3": 10,
"epochs_4": 10,
"max_lr_s1": 0.003,
"max_lr_s2": slice(1e-5, 3e-4),
"max_lr_s1_448": 0.003,
"max_lr_s2_448":slice(1e-5, 3e-4)
 },
"dense121": {
"model": models.densenet121,
"epochs_1": 6,
"epochs_2": 5,
"epochs_3": 6,
"epochs_4": 3,
"max_lr_s1": 0.003,
"max_lr_s2": slice(1e-5, 3e-4),
"max_lr_s1_448": 0.003,
"max_lr_s2_448":slice(1e-5, 1e-4)
 },
"dense161": {
"model": models.densenet161,
"epochs_1": 8,
"epochs_2": 5,
"epochs_3": 7,
"epochs_4": 4,
"max_lr_s1": 1e-3,
"max_lr_s2": slice(1e-5, 1e-4),
"max_lr_s1_448": 0.003,
"max_lr_s2_448":slice(1e-5, 3e-4)
 },
"eff-b3": {
"model": getModelEff('efficientnet-b3'),
"epochs_1": 7,
"epochs_2": 3,
"epochs_3": 5,
"epochs_4": 2,
"max_lr_s1": 1e-3,
"max_lr_s2": slice(1e-5, 1e-4),
"max_lr_s1_448": 1e-3,
"max_lr_s2_448":slice(1e-5, 3e-4)
 },
"eff-b4": {
"model": getModelEff('efficientnet-b4'),
"epochs_1": 10,
"epochs_2": 5,
"epochs_3": 8,
"epochs_4": 5,
"max_lr_s1": 3e-3,
"max_lr_s2": slice(1e-5, 3e-4),
"max_lr_s1_448": 1e-3,
"max_lr_s2_448":slice(1e-5, 1e-4)
 }
}<init_hyperparams>
df.Open.value_counts()
archs = {
"resnet152": {
"model_name": "resnet152",
"model": models.resnet152,
"stages": [{
"name_s1": "stage-1_lr",
"epochs_s1": 10,
"max_lr_s1": 1e-3,
"name_s2": "stage-2_lr",
"epochs_s2": 10,
"max_lr_s2": slice(1e-5, 1e-4),
"size": 224,
"bs": 20
},
{
"name_s1": "stage-1_mr",
"epochs_s1": 10,
"max_lr_s1": 1e-3,
"name_s2": "stage-2_mr",
"epochs_s2": 10,
"max_lr_s2": slice(1e-5, 1e-4),
"size": 448,
"bs": 10
},
{
"name_s1": "stage-1_hr",
"epochs_s1": 10,
"max_lr_s1": 1e-3,
"name_s2": "stage-2_hr",
"epochs_s2": 10,
"max_lr_s2": slice(1e-5, 1e-4),
"size": 896,
"bs": 5
}]
}
}<choose_model_class>
np.sum([df['Sales'] == 0] )
def train_model(arch):
model_name = arch["model_name"]
print(arch)
 for i, stage in enumerate(arch["stages"]):
print(f"traing {model_name} on {stage['name_s1']}")
 data = get_data(size=stage["size"], bs=stage["bs"])
 if model_name.startswith("eff"):
learner = Learner(data, arch["model"], metrics=error_rate ).to_fp16()
learner.split(lambda m:(arch._conv_head,))
else:
learner = cnn_learner(data, arch["model"], metrics=error_rate ).to_fp16()
learner.model_dir = "/kaggle/working"
if i != 0:
learner.load(f"{arch['stages'][i-1]['name_s2']}_{model_name}")
 print(f"lr for {model_name} {stage['name_s1']}")
 learner.lr_find()
learner.recorder.plot(suggestion=True)
 learner.fit_one_cycle(stage['epochs_s1'], max_lr=stage['max_lr_s1'])
 learner.save(f"{stage['name_s1']}_{model_name}")
 print(f"lr for {model_name} {stage['name_s2']}")
 learner.unfreeze()
learner.lr_find()
learner.recorder.plot(suggestion=True)
 learner.fit_one_cycle(stage['epochs_s2'], max_lr=stage['max_lr_s2'])
 learner.save(f"{stage['name_s2']}_{model_name}")
 del data
gc.collect()
if i == len(arch["stages"])-1:
return learner
learner.destroy()
torch.cuda.empty_cache()<load_from_csv>
df = df[(df["Open"] != 0)&(df['Sales'] != 0)]
def create_results() :
test_images = ImageList.from_folder(base_path/"images")
 test_images.filter_by_func(lambda x: x.name.startswith("Test"))
for model_name in archs:
arch = archs[model_name]
learner = train_model(arch)
 test_df = pd.read_csv(base_path/"test.csv")
 test_df['healthy'] = [0.0 for _ in test_df.index]
test_df['multiple_diseases'] = [0.0 for _ in test_df.index]
test_df['rust'] = [0.0 for _ in test_df.index]
test_df['scab'] = [0.0 for _ in test_df.index]
test_df = test_df.set_index('image_id')
 for item in tqdm(test_images.items):
name = item.name[:-4]
img = open_image(item)
 preds = learner.predict(img)[2]
test_df.loc[name]['healthy'] = preds[0]
test_df.loc[name]['multiple_diseases'] = preds[1]
test_df.loc[name]['rust'] = preds[2]
test_df.loc[name]['scab'] = preds[3]
test_df.to_csv(f"/kaggle/working/{model_name}_result.csv")
 learner.destroy()
torch.cuda.empty_cache()<load_from_csv>
store = pd.read_csv('.. /input/rossmann-store-sales/store.csv')
 store.head(30 )
sub1 = pd.read_csv('.. /input/tpu-incepresnetv2-enb7/submission.csv')
 sub1.head()<load_from_csv>
store.isnull().sum()
sub2 = pd.read_csv('.. /input/average-efficientnet/submission.csv')
 sub2.head()<load_from_csv>
store['CompetitionDistance'] = store['CompetitionDistance'].fillna(store['CompetitionDistance'].max())
 store['CompetitionOpenSinceMonth'] = store['CompetitionOpenSinceMonth'].fillna(store['CompetitionOpenSinceMonth'].mode().iloc[0])
 store['CompetitionOpenSinceYear'] = store['CompetitionOpenSinceYear'].fillna(store['CompetitionOpenSinceYear'].mode().iloc[0])
 store['Promo2SinceWeek'] = store['Promo2SinceWeek'].fillna(0)
 store['Promo2SinceYear'] = store['Promo2SinceYear'].fillna(store['Promo2SinceYear'].mode().iloc[0])
 store['PromoInterval'] = store['PromoInterval'].fillna(store['PromoInterval'].mode().iloc[0])
 store.head()
sub3 = pd.read_csv('.. /input/classification-densenet201-efficientnetb7/submission.csv')
 sub3.head()<load_from_csv>
df_store = pd.merge(df, store, how = 'left', on = 'Store')
 df_store.head()
sub4 = pd.read_csv('.. /input/plant-pathology-pytorch-gpu-model/v6_efficientnet_b4_5_fold_submission1_fold_0_39ep_best_fold_1_30ep_best_fold_2_33ep_best_fold_3_25ep_best_fold_4_44ep_best.csv')
 sub4.head()<load_from_csv>
df_store.groupby('StoreType')['Customers', 'Sales'].sum()
sub5 = pd.read_csv('.. /input/tf-zoo-models-on-tpu/submission.csv')
 sub5.head()<load_from_csv>
df_store['StateHoliday'] = df_store['StateHoliday'].map({'0':0 , 0:0 , 'a':1 , 'b':2 , 'c':3})
 df_store['StateHoliday'] = df_store['StateHoliday'].astype(int )
sub6 = pd.read_csv('.. /input/fork-of-plant-2020-tpu-915e9c/submission.csv')
 sub6.head()<load_from_csv>
df_store['StoreType'] = df_store['StoreType'].map({'a':1 , 'b':2 , 'c':3 , 'd':4})
 df_store['StoreType'] = df_store['StoreType'].astype(int )
sub = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/sample_submission.csv')
 sub.head()<feature_engineering>
df_store.isnull().sum()
sub.healthy =(sub1.healthy + sub2.healthy + sub3.healthy + sub4.healthy + sub5.healthy + sub6.healthy)/6<define_variables>
df_store['Assortment'] = df_store['Assortment'].map({'a':1 , 'b':2 , 'c':3})
 df_store['Assortment'] = df_store['Assortment'].astype(int )
sub.multiple_diseases =(sub1.multiple_diseases + sub2.multiple_diseases + sub3.multiple_diseases + sub4.multiple_diseases + sub5.multiple_diseases + sub6.multiple_diseases)/6<define_variables>
df_store['PromoInterval'] = df_store['PromoInterval'].map({'Jan,Apr,Jul,Oct':1 , 'Feb,May,Aug,Nov':2 , 'Mar,Jun,Sept,Dec':3})
 df_store['PromoInterval'] = df_store['PromoInterval'].astype(int )
sub.rust =(sub1.rust + sub2.rust + sub3.rust + sub4.rust + sub5.rust + sub6.rust)/6<save_to_csv>
df_store.to_csv('df_merged.csv', index=False )
sub.to_csv('submission.csv', index=False )<save_to_csv>
df_store.isnull().sum()
sub.to_csv('submission.csv', index=False )<import_modules>
test = pd.merge(test, store, how = 'left', on = 'Store')
 test.head()
print(tf.__version__)
 print(tf.keras.__version__ )<install_modules>
test.isnull().sum()
!pip install efficientnet 
<set_options>
test.isnull().sum()
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()<train_model>
test.fillna(method='ffill', inplace=True )
img = plt.imread('.. /input/plant-pathology-2020-fgvc7/images/Train_0.jpg')
 print(img.shape)
 plt.imshow(img )<load_from_csv>
test['StateHoliday'] = test['StateHoliday'].map({'0':0 , 0:0 , 'a':1 , 'b':2 , 'c':3})
 test['StateHoliday'] = test['StateHoliday'].astype(int)
 test['StoreType'] = test['StoreType'].map({'a':1 , 'b':2 , 'c':3 , 'd':4})
 test['StoreType'] = test['StoreType'].astype(int)
 test['Assortment'] = test['Assortment'].map({'a':1 , 'b':2 , 'c':3})
 test['Assortment'] = test['Assortment'].astype(int)
 test['PromoInterval'] = test['PromoInterval'].map({'Jan,Apr,Jul,Oct':1 , 'Feb,May,Aug,Nov':2 , 'Mar,Jun,Sept,Dec':3})
 test['PromoInterval'] = test['PromoInterval'].astype(int )
path='.. /input/plant-pathology-2020-fgvc7/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'sample_submission.csv')
 train_paths = train.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
test_paths = test.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
train_labels = train.loc[:, 'healthy':].values<define_variables>
test.to_csv('test_merged.csv', index=False )
nb_classes = 4
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
img_size = 768
EPOCHS = 40<normalization>
test = test.drop(['Id','Date'],axis=1 )
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label<categorify>
X = df_store.drop(['Date','Sales','Customers', 'SalesPerCustomer'],1)
 y = np.log1p(df_store['Sales'])
 X_train , X_val , y_train , y_val = train_test_split(X, y , test_size=0.30 , random_state = 1 )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<categorify>
gbrt = GradientBoostingRegressor(max_depth=10, n_estimators=200, random_state=42)
 gbrt.fit(X_train, y_train)
 print(gbrt.score(X_train, y_train))
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<init_hyperparams>
y_pred = gbrt.predict(X_val )
LR_START = 0.00001
LR_MAX = 0.0001 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 15
LR_SUSTAIN_EPOCHS = 3
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
print(r2_score(y_val , y_pred))
print(np.sqrt(mean_squared_error(y_val , y_pred)) )
def get_model() :
base_model = efn.EfficientNetB7(weights='imagenet', include_top=False, pooling='avg', input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = Dense(nb_classes, activation="softmax" )(x)
 return Model(inputs=base_model.input, outputs=predictions )<choose_model_class>
df1 = pd.DataFrame({'Actual': y_val, 'Predicted': y_pred})
 df1.head(25 )
with strategy.scope() :
model = get_model()
model.compile(optimizer='adam', loss='categorical_crossentropy',metrics=['accuracy'] )<train_model>
test_pred=gbrt.predict(test[X.columns])
 test_pred_inv=np.exp(test_pred)-1
<predict_on_test><EOS> 
prediction = pd.DataFrame(test_pred_inv)
 submission = pd.read_csv('.. /input/rossmann-store-sales/sample_submission.csv')
 prediction_df = pd.concat([submission['Id'], prediction], axis=1)
 prediction_df.columns=['Id','Sales']
prediction_df.to_csv('Sample_Submission.csv', index=False )