Kaggle competition: Rossmann Store Sales
<SOS> metric: RootMeanSquarePercentageError Kaggle data source: rossmann-store-sales<choose_model_class>
%matplotlib inline
%reload_ext autoreload
%autoreload 2
np.random.seed(23)
 np.set_printoptions(threshold=50, edgeitems=20 )
INFINITY = 100
lr = LogisticRegression(C=INFINITY )<merge>
OUTPUT = '/kaggle/working/'
PATH='/kaggle/input/rossmann-time-series-data-engineering/'
df = pd.read_feather(f'{PATH}df')
 train_df = pd.read_feather(f'{PATH}joined2')
 test_df = pd.read_feather(f'{PATH}joined2_test')
 train_df.shape, test_df.shape
test_chex = tc0.rename(columns={'targetPredProba':'pg0'} ).drop(['targetPred'],axis=1)
 test_chex = test_chex.join(tc1.rename(columns={'targetPredProba':'pg1'} ).drop(['targetPred'],axis=1))
test_chex = test_chex.join(tc2.rename(columns={'targetPredProba':'pg2'} ).drop(['targetPred'],axis=1))
test_chex = test_chex.join(tc3.rename(columns={'targetPredProba':'pg3'} ).drop(['targetPred'],axis=1))
test_chex = test_chex.join(tc4.rename(columns={'targetPredProba':'pg4'} ).drop(['targetPred'],axis=1))
print(test_chex.shape)
 test_chex.head()<merge>
cat_vars = ['Store', 'DayOfWeek', 'Promo',
'StateHoliday', 'SchoolHoliday', 'Year', 'Month', 'Week', 'Day',
'Is_year_end', 'Is_year_start', 'StoreType', 'Assortment',
'Promo2', 'PromoInterval', 'State',
'Events', 'CompetitionMonthsOpen',
'Promo2Weeks',
'SchoolHoliday_bw','StateHoliday_bw', 'Promo_bw', 'SchoolHoliday_fw', 'StateHoliday_fw','Promo_fw',
'SchoolHoliday_DaySum', 'StateHoliday_DaySum', 'Promo_DaySum',
'SchoolHoliday_DayCount', 'StateHoliday_DayCount', 'Promo_DayCount']
cont_vars = ['CompetitionDistance', 'Max_TemperatureC', 'Mean_TemperatureC', 'Min_TemperatureC',
'Max_Humidity','Mean_Humidity', 'Min_Humidity',
'Max_Wind_SpeedKm_h', 'Mean_Wind_SpeedKm_h','Precipitationmm','CloudCover',
'trend', 'trend_DE', 'CompetitionDaysOpen', 'Promo2Days',
'AfterSchoolHoliday', 'BeforeSchoolHoliday', 'AfterStateHoliday',
'BeforeStateHoliday', 'AfterPromo', 'BeforePromo']
dep_var = 'Sales'
df = train_df[cat_vars + cont_vars + [dep_var,'Date']].copy()
all_test_probs = test_prob_df.join(test_den.join(test_chex))
all_test_probs.head()<define_variables>
cut = train_df['Date'][(train_df['Date'] == train_df['Date'][len(test_df)])].index.max()
valid_idx = range(cut); valid_idx
test_sets = []
test_set = ['pa','pg','ph']
for i in range(5):
this_set = [n + str(i)for n in test_set]
test_sets.append(this_set)
 test_sets<predict_on_test>
procs=[FillMissing, Categorify, Normalize]
datalist =(TabularList.from_df(df, path=OUTPUT, cat_names=cat_vars, cont_names=cont_vars, procs=procs,)
.split_by_idx(valid_idx=valid_idx)
.label_from_df(cols=dep_var, label_cls=FloatList, log=True)
.add_test(TabularList.from_df(test_df, path=PATH, cat_names=cat_vars, cont_names=cont_vars)))
 data = datalist.databunch(bs=512 )
yps = []
MAXCONF =.996
Xtr = df.drop('actual',axis=1 ).copy()
ytr = df.actual.copy()
X_train_adjust = Xtr
X_train_adjust.loc[Xtr.ph>MAXCONF,'ph'] = MAXCONF
X_train_logit = X_train_adjust.apply(logit)
 lr.fit(X_train_logit, ytr)
 for tset in test_sets:
Xte = all_test_probs[tset].copy()
Xte.columns = test_set
X_test_adjust = Xte
X_test_adjust.loc[Xte.ph>MAXCONF,'ph'] = MAXCONF
X_test_logit = X_test_adjust.apply(logit)
 yp = lr.predict_proba(X_test_logit)[:,1]
yps.append(yp)
 len(yps), [len(y)for y in yps]<prepare_output>
defaults.device
p = pd.DataFrame(yps,columns=Xte.index ).transpose().apply(logit ).mean(axis=1 ).apply(expit)
 class_preds = pd.DataFrame(p,columns=['prob'])
 class_preds.index.name = 'patientId'
print(class_preds.shape)
 class_preds.head()<prepare_output>
max_log_y = np.log(np.max(train_df['Sales'])*1.2)
 y_range = torch.tensor([0, max_log_y], device=defaults.device)
 learn = tabular_learner(data, layers=[1000,500], ps=[0.001,0.01], emb_drop=0.04,
y_range=y_range, metrics=exp_rmspe )
phase4_test_output = class_preds<prepare_x_and_y>
learn.to_fp16
yps = []
ids = []
MAXCONF =.995
folds = [p0,p1,p2,p3,p4]
for i in range(len(folds)) :
fs = folds.copy()
te = fs.pop(i)
 tr = np.concatenate(fs)
 Xtr = df.loc[tr,:].drop('actual',axis=1 ).copy()
Xte = df.loc[te,:].drop('actual',axis=1 ).copy()
ytr = df.actual[tr].copy()
yte = df.actual[te].copy()
X_train_adjust = Xtr
X_test_adjust = Xte
X_train_adjust.loc[Xtr.ph>MAXCONF,'ph'] = MAXCONF
X_test_adjust.loc[Xte.ph>MAXCONF,'ph'] = MAXCONF
X_train_logit = X_train_adjust.apply(logit)
 X_test_logit = X_test_adjust.apply(logit)
 lr.fit(X_train_logit, ytr)
 yp = lr.predict_proba(X_test_logit)[:,1]
yps = yps + list(yp)
 ids = ids + list(Xte.index.values )<prepare_output>
learn.data.batch_size
oof_class_preds = pd.DataFrame({'prob':yps},index=ids)
 oof_class_preds.index.name = 'patientId'
print(oof_class_preds.shape)
 oof_class_preds.head()<define_variables>
fastai.__version__
phase4_oof_output = oof_class_preds<load_from_csv>
learn.lr_find(end_lr=100, wd=0.3)
 learn.recorder.plot()
df1 = phase2_output
df2 = pd.read_csv(UNET_RESULTS_PATH ).rename(columns={
'predictionString':'PredictionString'} ).set_index('patientId')
 class_preds = phase4_test_output
class_probs = class_preds.prob.to_dict()
df1_cases = df1[~df1.PredictionString.isnull() ]
df2_cases = df2[~df2.PredictionString.isnull() ]
df1_pos_ids = df1_cases.index.values
df2_pos_ids = df2_cases.index.values<feature_engineering>
learn.fit_one_cycle(5, 3e-3, wd=0.3 )
df1_only_dict = {}
for p in df1_pos_ids:
if not p in df2_pos_ids:
df1_only_dict[p] = float(df1_cases.loc[p,'PredictionString'].split(' ')[0])
 df2_only_dict = {}
for p in df2_pos_ids:
if not p in df1_pos_ids:
df2_only_dict[p] = float(df2_cases.loc[p,'PredictionString'].split(' ')[0])
 candidates = pd.DataFrame(pd.Series(df2_only_dict,name='conf')).join(class_preds,how='left')
 accepted = candidates[(candidates.prob>UNET_MINPROB)&(candidates.conf>UNET_MINCONF)
 ].index.values<feature_engineering>
learn.save('bs512_5ep_2e-3_wd0.3' )
df_out = df1.copy()
for p,r in df1.iterrows() :
if p in accepted:
df_out.loc[p,'PredictionString'] = df2.loc[p,'PredictionString']
phase5_output = df_out
phase5_output.head()<define_search_space>
learn.fit_one_cycle(5, 1e-3, wd=0.3 )
yolo_epochs = [1500, 2000, 3200, 4100, 5300, 6700, 10000, 13700]<load_from_csv>
learn.save('bs512_2_5ep_1e-3_wd0.3' )
df1 = phase5_output
dfs = []
for eps in yolo_epochs:
yolo_stem = YOLO_SUB_STEM2 if(eps in MORE_YOLO_EPOCHS)else YOLO_SUB_STEM
dfs.append(
 pd.read_csv(RESULTS_LOC+'/'+yolo_stem+str(eps)+'.csv' ).set_index('patientId'))
class_preds = phase4_test_output
class_probs = class_preds.prob.to_dict()
df1_cases = df1[~df1.PredictionString.isnull() ]
df1_pos_ids = df1_cases.index.values
included = df1_pos_ids.tolist()
df_out = df1.copy()<define_variables>
learn.fit_one_cycle(5, 3e-4, wd=0.3)
 learn.recorder.plot_losses()
while True:
for df2 in dfs:
df2_cases = df2[~df2.PredictionString.isnull() ]
df2_pos_ids = df2_cases.index.values
df2_only_dict = {}
outstring_dict = {}
for p in df2_pos_ids:
if not p in included:
maxconf = 0
boxes = []
confs = []
s = df2.loc[p,'PredictionString'].split(' ')
 if s[-1]=='':
s.pop()
if s[0]=='':
s.pop(0)
 if(len(s)%5):
print('Bad prediction string.')
 while len(s):
conf = float(s.pop(0))
x = int(round(float(s.pop(0))))
y = int(round(float(s.pop(0))))
w = int(round(float(s.pop(0))))
h = int(round(float(s.pop(0))))
if(conf>YOLO_MINCONF):
boxes.append([x,y,w,h])
 confs.append(conf)
 if conf>maxconf:
maxconf = conf
predictionString = ''
if len(boxes):
for box, conf in zip(boxes, confs):
x, y, w, h = box
predictionString += '{:6.4f} '.format(conf)
 predictionString += str(x)+ ' ' + str(y)+ ' ' + str(w)+ ' ' + str(h)+ ' '
df2_only_dict[p] = maxconf*class_probs[p]
outstring_dict[p] = predictionString
candidates = pd.DataFrame(pd.Series(df2_only_dict,name='prod'))
best = candidates.sort_values('prod' ).tail(1 ).index.values[0]
print(best,candidates.loc[best,'prod'])
 df_out.loc[best,'PredictionString'] = outstring_dict[best]
included.append(best)
 minprob = class_preds.loc[included[-8:],:].prob.min()
if minprob<MINMIN:
break<filter>
learn.save('bs512_3_5ep_3e-4_wd0.3' )
class_preds.loc[included[-32:],:]<save_to_csv>
data = datalist.databunch(bs=128)
 learn.data = data
learn.data.batch_size
df_out.to_csv('phase6_output.csv' )<import_modules>
learn.fit_one_cycle(5, 1e-3, wd=0.2)
 learn.recorder.plot_losses()
import os
import sys
import random
import math
import numpy as np
import cv2
import matplotlib.pyplot as plt
import json
import pydicom
from imgaug import augmenters as iaa
from tqdm import tqdm
import pandas as pd
import glob<define_variables>
learn.save('bs128_5ep_1e-3_wd0.2' )
DATA_DIR = '/kaggle/input'
ROOT_DIR = '/kaggle/working'<load_from_zip>
learn.fit_one_cycle(5, 1e-3, wd=0.2)
 learn.recorder.plot_losses()
!git clone https://www.github.com/matterport/Mask_RCNN.git
os.chdir('Mask_RCNN')
<define_variables>
learn.fit_one_cycle(5, 1e-3, wd=0.1)
 learn.recorder.plot_losses()
train_dicom_dir = os.path.join(DATA_DIR, 'stage_1_train_images')
 test_dicom_dir = os.path.join(DATA_DIR, 'stage_1_test_images' )<create_dataframe>
learn.fit_one_cycle(20, 1e-3, wd=0.1)
 learn.recorder.plot_losses()
def get_dicom_fps(dicom_dir):
dicom_fps = glob.glob(dicom_dir+'/'+'*.dcm')
 return list(set(dicom_fps))
def parse_dataset(dicom_dir, anns):
image_fps = get_dicom_fps(dicom_dir)
 image_annotations = {fp: [] for fp in image_fps}
for index, row in anns.iterrows() :
fp = os.path.join(dicom_dir, row['patientId']+'.dcm')
 image_annotations[fp].append(row)
 return image_fps, image_annotations<init_hyperparams>
learn.save('last' )
class DetectorConfig(Config):

 NAME = 'pneumonia'
GPU_COUNT = 1
IMAGES_PER_GPU = 8
BACKBONE = 'resnet50'
NUM_CLASSES = 2
IMAGE_MIN_DIM = 256
IMAGE_MAX_DIM = 256
RPN_ANCHOR_SCALES =(32, 64, 128, 256)
 TRAIN_ROIS_PER_IMAGE = 32
MAX_GT_INSTANCES = 3
DETECTION_MAX_INSTANCES = 3
DETECTION_MIN_CONFIDENCE = 0.7
DETECTION_NMS_THRESHOLD = 0.1
STEPS_PER_EPOCH = 200
config = DetectorConfig()
config.display()<categorify>
test_preds=learn.get_preds(DatasetType.Test)
 test_df["Sales"]=np.exp(test_preds[0].data ).numpy().T[0]
test_df[["Id","Sales"]]=test_df[["Id","Sales"]].astype("int")
 test_df[["Id","Sales"]].to_csv("rossmann_submission.csv",index=False )