Kaggle competition: Rossmann Store Sales
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'<load_from_csv>
train = pd.read_csv(".. /input/train.csv",parse_dates=[2], low_memory=False)
 test = pd.read_csv(".. /input/test.csv",parse_dates=[3], low_memory=False)
 store = pd.read_csv(".. /input/store.csv", low_memory=False )
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.15, random_state=2020 )<normalization>
train.head().append(train.tail() )
def decode_image(filename, label=None, image_size=(512, 512)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label<categorify>
test.head().append(test.tail() )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<choose_model_class>
display(train.isnull().sum() ,test.isnull().sum() ,store.isnull().sum() )
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn<choose_model_class>
test[pd.isnull(test.Open)]
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model.summary()<train_model>
store[pd.isnull(store.CompetitionDistance)]
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset
)<save_to_csv>
test.fillna(1, inplace=True)
probs = model.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()<import_modules>
store.fillna(0, inplace=True )
!pip install -q efficientnet 
<load_from_csv>
display(train.isnull().sum() ,test.isnull().sum() ,store.isnull().sum() )
train_df = pd.read_csv(".. /input/plant-pathology-2020-fgvc7/train.csv")
 IMAGE_PATH = ".. /input/plant-pathology-2020-fgvc7/images/"<count_values>
train = pd.merge(train, store, on='Store')
 test = pd.merge(test, store, on='Store' )
print(train_df['healthy'].value_counts())
 print(train_df['multiple_diseases'].value_counts())
 print(train_df['rust'].value_counts())
 print(train_df['scab'].value_counts() )<filter>
train = train.sort_values(['Date'],ascending = False)
 ho_test = train[:6*7*1115]
ho_train = train[6*7*1115:]
healthy = list(train_df[train_df["healthy"]==1].image_id)
 multiple_diseases = list(train_df[train_df["multiple_diseases"]==1].image_id)
 rust = list(train_df[train_df["rust"]==1].image_id)
 scab = list(train_df[train_df["scab"]==1].image_id )<load_pretrained>
ho_test = ho_test[ho_test["Open"] != 0]
ho_test = ho_test[ho_test["Sales"] > 0]
ho_train = ho_train[ho_train["Open"] != 0]
ho_train = ho_train[ho_train["Sales"] > 0]
def load_image(filenames):
sample = random.choice(filenames)
 image = load_img(".. /input/plant-pathology-2020-fgvc7/images/"+sample+".jpg")
 plt.imshow(image )<load_pretrained>
def features_create(data):
mappings = {'0':0, 'a':1, 'b':2, 'c':3, 'd':4}
data.StoreType.replace(mappings, inplace=True)
 data.Assortment.replace(mappings, inplace=True)
 data.StateHoliday.replace(mappings, inplace=True)
 data['Year'] = data.Date.dt.year
data['Month'] = data.Date.dt.month
data['Day'] = data.Date.dt.day
data['DayOfWeek'] = data.Date.dt.dayofweek
data['WeekOfYear'] = data.Date.dt.weekofyear
data['CompetitionOpen'] = 12 *(data.Year - data.CompetitionOpenSinceYear)+(data.Month - data.CompetitionOpenSinceMonth)
 data['PromoOpen'] = 12 *(data.Year - data.Promo2SinceYear)+(data.WeekOfYear - data.Promo2SinceWeek)/ 4.0
data['CompetitionOpen'] = data.CompetitionOpen.apply(lambda x: x if x > 0 else 0)
 data['PromoOpen'] = data.PromoOpen.apply(lambda x: x if x > 0 else 0)
 month2str = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug', 9:'Sept', 10:'Oct', 11:'Nov', 12:'Dec'}
data['monthStr'] = data.Month.map(month2str)
 data.loc[data.PromoInterval == 0, 'PromoInterval'] = ''
data['IsPromoMonth'] = 0
for interval in data.PromoInterval.unique() :
if interval != '':
for month in interval.split(','):
data.loc[(data.monthStr == month)&(data.PromoInterval == interval), 'IsPromoMonth'] = 1
return data
load_image(healthy )<train_model>
features_create(ho_train)
 features_create(ho_test)
 features_create(test)
 print('Features creation finished' )
load_image(multiple_diseases )<load_pretrained>
ho_train.drop(['Date','Customers','Open','PromoInterval','monthStr'],axis=1,inplace =True)
 ho_test.drop(['Date','Customers','Open','PromoInterval','monthStr'],axis=1,inplace =True )
load_image(rust )<train_model>
ho_xtrain = ho_train.drop(['Sales'],axis=1)
 ho_ytrain = np.log1p(ho_train.Sales)
 ho_xtest = ho_test.drop(['Sales'],axis=1)
 ho_ytest = np.log1p(ho_test.Sales )
load_image(scab )<train_model>
xtest =test.drop(['Id','Date','Open','PromoInterval','monthStr'],axis = 1 )
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
!gsutil ls $GCS_DS_PATH<split>
def rmspe(y, yhat):
return np.sqrt(np.mean(( yhat/y-1)** 2))
def rmspe_xg(yhat, y):
y = np.expm1(y.get_label())
 yhat = np.expm1(yhat)
 return "rmspe", rmspe(y,yhat )
def format_path_gcs(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
X = train_df.image_id.apply(format_path_gcs ).values
y = np.float32(train_df.loc[:, 'healthy':'scab'].values)
 X_train, X_val, y_train, y_val =train_test_split(X, y, test_size=0.1, random_state=43)
 print('done!' )<set_options>
params = {"objective": "reg:linear",
"booster" : "gbtree",
"eta": 0.03,
"max_depth": 10,
"subsample": 0.9,
"colsample_bytree": 0.7,
"silent": 1,
"seed": 10
}
num_boost_round = 6000
dtrain = xgb.DMatrix(ho_xtrain, ho_ytrain)
 dvalid = xgb.DMatrix(ho_xtest, ho_ytest)
 watchlist = [(dtrain, 'train'),(dvalid, 'eval')]
print("Train a XGBoost model")
 start = time()
gbm = xgb.train(params, dtrain, num_boost_round, evals=watchlist,
early_stopping_rounds=100, feval=rmspe_xg, verbose_eval=True)
 end = time()
print('Training time is {:2f} s.'.format(end-start))
print("validating")
 ho_xtest.sort_index(inplace=True)
 ho_ytest.sort_index(inplace=True)
 yhat = gbm.predict(xgb.DMatrix(ho_xtest))
error = rmspe(np.expm1(ho_ytest), np.expm1(yhat))
print('RMSPE: {:.6f}'.format(error))
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
res = pd.DataFrame(data = ho_ytest)
 res['Prediction']=yhat
res = pd.merge(ho_xtest,res, left_index= True, right_index=True)
 res['Ratio'] = res.Prediction/res.Sales
res['Error'] =abs(res.Ratio-1)
 res['Weight'] = res.Sales/res.Prediction
res.head()
BATCH_SIZE = 4 * strategy.num_replicas_in_sync
STEPS_PER_EPOCH = y_train.shape[0] // BATCH_SIZE<normalization>
col_1 = ['Sales','Prediction']
col_2 = ['Ratio']
L=np.random.randint(low=1,high = 1115, size = 3)
 print('Mean Ratio of predition and real sales data is {}: store all'.format(res.Ratio.mean()))
for i in L:
s1 = pd.DataFrame(res[res['Store']==i],columns = col_1)
 s2 = pd.DataFrame(res[res['Store']==i],columns = col_2)
 s1.plot(title = 'Comparation of predition and real sales data: store {}'.format(i),figsize=(12,4))
s2.plot(title = 'Ratio of predition and real sales data: store {}'.format(i),figsize=(12,4))
print('Mean Ratio of predition and real sales data is {}: store {}'.format(s2.Ratio.mean() ,i))
def decode_image(filename, label=None, image_size=(1024,1024)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 image = tf.image.random_brightness(image, max_delta=0.3)
 if label is None:
return image
else:
return image, label<train_model>
res.sort_values(['Error'],ascending=False,inplace= True)
 res[:10]
train_dataset =(
 tf.data.Dataset.from_tensor_slices(( X_train, y_train)) 
.map(decode_image,num_parallel_calls=AUTO)
.map(data_augment,num_parallel_calls=AUTO)
.repeat() 
.shuffle(256)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( X_val, y_val)) 
.map(decode_image,num_parallel_calls=AUTO)
.cache() 
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<choose_model_class>
L=range(1115)
 W_ho=[]
W_test=[]
for i in L:
s1 = pd.DataFrame(res[res['Store']==i+1],columns = col_1)
 s2 = pd.DataFrame(xtest[xtest['Store']==i+1])
 W1=[(0.990+(i/1000)) for i in range(20)]
S =[]
for w in W1:
error = rmspe(np.expm1(s1.Sales), np.expm1(s1.Prediction*w))
S.append(error)
 Score = pd.Series(S,index=W1)
 BS = Score[Score.values == Score.values.min() ]
a=np.array(BS.index.values)
 b_ho=a.repeat(len(s1))
b_test=a.repeat(len(s2))
W_ho.extend(b_ho.tolist())
 W_test.extend(b_test.tolist() )
def build_lrfn(lr_start=0.00001, lr_max=0.00005,lr_min=0.00001, lr_rampup_epochs=5,lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)*\
lr_exp_decay**(epoch - lr_rampup_epochs- lr_sustain_epochs)+ lr_min
return lr
return lrfn<choose_model_class>
yhat_new = yhat*W_ho
error = rmspe(np.expm1(ho_ytest), np.expm1(yhat_new))
print('RMSPE for weight corretion {:6f}'.format(error))
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 EarlyStopping=tf.keras.callbacks.EarlyStopping(monitor="val_loss",patience=10,verbose=True, mode="min" )<choose_model_class>
print("Make predictions on the test set")
 dtest = xgb.DMatrix(xtest)
 test_probs = gbm.predict(dtest)
 result = pd.DataFrame({"Id": test['Id'], 'Sales': np.expm1(test_probs)})
 result.to_csv("Rossmann_submission_1.csv", index=False)
 result = pd.DataFrame({"Id": test['Id'], 'Sales': np.expm1(test_probs*0.995)})
 result.to_csv("Rossmann_submission_2.csv", index=False)
 result = pd.DataFrame({"Id": test['Id'], 'Sales': np.expm1(test_probs*W_test)})
 result.to_csv("Rossmann_submission_3.csv", index=False)
def Eff_B7_NS() :
model_EfficientNetB7 = Sequential([efn.EfficientNetB7(input_shape=(1024,1024,3),weights='noisy-student',include_top=False),
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(128,activation='relu'),
tf.keras.layers.Dense(64,activation='relu'),
tf.keras.layers.Dense(4,activation='softmax')])
 model_EfficientNetB7.compile(optimizer='Adam',loss = 'categorical_crossentropy',metrics=['categorical_accuracy'])
 return model_EfficientNetB7<train_model>
EfficientNetB7 = model_Eff_B7.fit(train_dataset,
epochs=20,
callbacks=[lr_schedule,EarlyStopping],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset )<load_from_csv>