Kaggle competition: Rossmann Store Sales
class DetectorDataset(utils.Dataset):

 def __init__(self, image_fps, image_annotations, orig_height, orig_width):
super().__init__(self)
 self.add_class('pneumonia', 1, 'Lung Opacity')
 for i, fp in enumerate(image_fps):
annotations = image_annotations[fp]
self.add_image('pneumonia', image_id=i, path=fp,
annotations=annotations, orig_height=orig_height, orig_width=orig_width)
 def image_reference(self, image_id):
info = self.image_info[image_id]
return info['path']
def load_image(self, image_id):
info = self.image_info[image_id]
fp = info['path']
ds = pydicom.read_file(fp)
 image = ds.pixel_array
if len(image.shape)!= 3 or image.shape[2] != 3:
image = np.stack(( image,)* 3, -1)
 return image
def load_mask(self, image_id):
info = self.image_info[image_id]
annotations = info['annotations']
count = len(annotations)
 if count == 0:
mask = np.zeros(( info['orig_height'], info['orig_width'], 1), dtype=np.uint8)
 class_ids = np.zeros(( 1,), dtype=np.int32)
 else:
mask = np.zeros(( info['orig_height'], info['orig_width'], count), dtype=np.uint8)
 class_ids = np.zeros(( count,), dtype=np.int32)
 for i, a in enumerate(annotations):
if a['Target'] == 1:
x = int(a['x'])
 y = int(a['y'])
 w = int(a['width'])
 h = int(a['height'])
 mask_instance = mask[:, :, i].copy()
cv2.rectangle(mask_instance,(x, y),(x+w, y+h), 255, -1)
 mask[:, :, i] = mask_instance
class_ids[i] = 1
return mask.astype(np.bool), class_ids.astype(np.int32 )<load_from_csv>
sales = pd.read_csv(".. /input/train.csv")
 sales.head()
anns = pd.read_csv(os.path.join(DATA_DIR, 'stage_1_train_labels.csv'))
anns.head()<create_dataframe>
stores = pd.read_csv(".. /input/store.csv")
 stores.head()
image_fps, image_annotations = parse_dataset(train_dicom_dir, anns=anns )<load_from_csv>
test = pd.read_csv('.. /input/test.csv')
 test = pd.merge(test, stores, 'left', 'Store')
 test.head()
ds = pydicom.read_file(image_fps[0])
 image = ds.pixel_array<define_variables>
data = pd.merge(sales, stores, 'left', 'Store' ).sample(frac= 1)
 data['Date'] = pd.to_datetime(data['Date'])
 data['Month'] = data['Date'].dt.month
data['Year'] = data['Date'].dt.year
data['Day'] = data['Date'].dt.day
data['WeekOfYear'] = data.Date.dt.weekofyear
data['Quarter'] = data.Date.dt.quarter
data['StateHoliday'] = data['StateHoliday'].replace(0, '0')
 data.head()
ORIG_SIZE = 1024<split>
print(len(set(sales['Store'])- set(stores['Store'])) )
image_fps_list = list(image_fps)
 random.seed(42)
 random.shuffle(image_fps_list)
 val_size = 1500
image_fps_val = image_fps_list[:val_size]
image_fps_train = image_fps_list[val_size:]
print(len(image_fps_train), len(image_fps_val))<create_dataframe>
print(len(set(test['Store'])- set(data['Store'])) )
dataset_train = DetectorDataset(image_fps_train, image_annotations, ORIG_SIZE, ORIG_SIZE)
 dataset_train.prepare()<create_dataframe>
print(len(set(data['Store'])- set(test['Store'])) )
dataset_val = DetectorDataset(image_fps_val, image_annotations, ORIG_SIZE, ORIG_SIZE)
 dataset_val.prepare()<load_pretrained>
zero_sales = data[data['Sales']==0].copy()
data = data[data['Sales']!=0].drop('Open', 1 )
model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)
 model.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[
"mrcnn_class_logits", "mrcnn_bbox_fc",
"mrcnn_bbox", "mrcnn_mask"] )<init_hyperparams>
int_cols = data.select_dtypes(include='int' ).drop(['Store', 'Customers', 'Sales'], 1 ).columns
categorical_cols = data.select_dtypes(include='object' ).columns
float_cols = data.select_dtypes(include='float' ).drop('CompetitionDistance', 1 ).columns
LEARNING_RATE = 0.005
warnings.filterwarnings("ignore" )<train_model>
data.fillna(0, inplace=True)
 test.fillna(1, inplace=True )
%%time
model.train(dataset_train, dataset_val,
learning_rate=LEARNING_RATE*2,
epochs=2,
layers='heads',
augmentation=None)
 history = model.keras_model.history.history.copy()<train_model>
data['part'] = 'train'
test['part'] = 'test'
all_data = pd.concat([data, test], 0)[data.columns.tolist() +['Id']]
all_data['Date'] = pd.to_datetime(all_data['Date'])
 all_data['Month'] = all_data['Date'].dt.month
all_data['Year'] = all_data['Date'].dt.year
all_data['Day'] = all_data['Date'].dt.day
all_data['WeekOfYear'] = data.Date.dt.weekofyear
all_data['Quarter'] = data.Date.dt.quarter
all_data.sort_values('Date', inplace=True )
%%time
model.train(dataset_train, dataset_val,
learning_rate=LEARNING_RATE,
epochs=7,
layers='all',
augmentation=augmentation)
 news = model.keras_model.history.history
for k in news: history[k] = history[k] + news[k]<train_model>
all_data['CompetitionOpen'] = 12 *(2015 - all_data.Year - all_data.CompetitionOpenSinceYear)+ \ 
(all_data.Month - all_data.CompetitionOpenSinceMonth)
 all_data['CompetitionOpen'] = all_data['CompetitionOpen'].apply(lambda x: x if x > 0 else 0 )
%%time
model.train(dataset_train, dataset_val,
learning_rate=LEARNING_RATE/5,
epochs=15,
layers='all',
augmentation=augmentation)
 news = model.keras_model.history.history
for k in news: history[k] = history[k] + news[k]<create_dataframe>
all_data['PromoOpen'] = 12 *(all_data.Year - all_data.Promo2SinceYear)+ \ 
(all_data.WeekOfYear - all_data.Promo2SinceWeek)/ 4.0
all_data['PromoOpen'] = all_data['PromoOpen'].apply(lambda x: x if x > 0 else 0 )
epochs = range(1,len(next(iter(history.values())))+1)
 pd.DataFrame(history, index=epochs )<load_pretrained>
month2str = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', \
7:'Jul', 8:'Aug', 9:'Sept', 10:'Oct', 11:'Nov', 12:'Dec'}
all_data['month_str'] = all_data.Month.map(month2str)
 def check(row):
if isinstance(row['PromoInterval'],str)and row['month_str'] in row['PromoInterval']:
return 1
else:
return 0
all_data['IsPromoMonth'] = all_data.apply(lambda row: check(row),axis=1 )
class InferenceConfig(DetectorConfig):
GPU_COUNT = 1
IMAGES_PER_GPU = 1
inference_config = InferenceConfig()
model = modellib.MaskRCNN(mode='inference',
config=inference_config,
model_dir=ROOT_DIR)
 assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
 model.load_weights(model_path, by_name=True )<find_best_params>
all_data ['isBeforeCompetition'] = all_data.apply(lambda x: 1 if x['Year'] < x['CompetitionOpenSinceYear'] else 0, 1 )
test_image_fps = get_dicom_fps(test_dicom_dir )<save_to_csv>
all_data['SalesPerCustomer'] = data['Sales']/data['Customers']
def predict(image_fps, filepath='submission.csv', min_conf=0.95):
resize_factor = ORIG_SIZE / config.IMAGE_SHAPE[0]
with open(filepath, 'w')as file:
file.write("patientId,PredictionString
")
 for image_id in tqdm(image_fps):
ds = pydicom.read_file(image_id)
 image = ds.pixel_array
if len(image.shape)!= 3 or image.shape[2] != 3:
image = np.stack(( image,)* 3, -1)
 image, window, scale, padding, crop = utils.resize_image(
 image,
min_dim=config.IMAGE_MIN_DIM,
min_scale=config.IMAGE_MIN_SCALE,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
 patient_id = os.path.splitext(os.path.basename(image_id)) [0]
results = model.detect([image])
 r = results[0]
out_str = ""
out_str += patient_id
out_str += ","
assert(len(r['rois'])== len(r['class_ids'])== len(r['scores']))
if len(r['rois'])== 0:
pass
else:
num_instances = len(r['rois'])
 for i in range(num_instances):
if r['scores'][i] > min_conf:
out_str += ' '
out_str += str(round(r['scores'][i], 2))
out_str += ' '
x1 = r['rois'][i][1]
y1 = r['rois'][i][0]
width = r['rois'][i][3] - x1
height = r['rois'][i][2] - y1
bboxes_str = "{} {} {} {}".format(x1*resize_factor, y1*resize_factor, \
width*resize_factor, height*resize_factor)
 out_str += bboxes_str
file.write(out_str+"
" )<predict_on_test>
mappings = {'0':0, 'a':1, 'b':2, 'c':3, 'd':4}
all_data.StoreType.replace(mappings, inplace=True)
 all_data.Assortment.replace(mappings, inplace=True)
 all_data.StateHoliday.replace(mappings, inplace=True )
submission_fp = os.path.join(ROOT_DIR, 'submission.csv')
 predict(test_image_fps, filepath=submission_fp)
 print(submission_fp )<load_from_csv>
def prepareDf(df, submission=False):
tests_date = "2015-06-12"
tmp_data = all_data[all_data['part']=='train'].copy()
if not submission:
tmp_data = tmp_data[tmp_data['Date']<tests_date]
shift = 48
cols = ['isBeforeCompetition', 'IsPromoMonth', 'PromoOpen', 'DayOfWeek', 'Promo', 'Promo2', 'Month', 'Year', 'Day', 'StateHoliday', 'Assortment', 'Quarter']
for i in ['Sales', 'Customers']:
for j in cols:
tmp = pd.pivot_table(tmp_data, i, ['Store', 'StoreType',j], aggfunc='mean' ).reset_index().rename(columns={i: i+'_StrType_'+j})
 df = pd.merge(df, tmp, 'left', ['Store','StoreType', j])
 cols = ['DayOfWeek', 'Promo', 'Promo2', 'Month', 'Year', 'Day', 'StateHoliday', 'StoreType', 'Assortment', 'PromoInterval', 'Quarter',
'isBeforeCompetition', 'IsPromoMonth', 'PromoOpen']
for i in ['Sales', 'Customers']:
for j in cols:
tmp = pd.pivot_table(tmp_data, i, ['Store', j], aggfunc='mean' ).reset_index().rename(columns={i: i+'_'+j})
 df = pd.merge(df, tmp, 'left', ['Store', j])
 df['rolling_mean_t7_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift ).rolling(7 ).mean())
 df['rolling_mean_t30_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift ).rolling(30 ).mean())
 df['rolling_mean_t360_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift ).rolling(360 ).mean())
 df['rolling_mean_t7_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift ).rolling(7 ).mean())
 df['rolling_mean_t30_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift ).rolling(30 ).mean())
 df['rolling_mean_t360_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift ).rolling(360 ).mean())
 df['shift_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift))
df['shift_t7_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift+7))
df['shift_t30_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift+30))
df['shift_t360_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift+360))
df['shift_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift))
df['shift_t7_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift+7))
df['shift_t30_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift+30))
df['shift_t360_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift+360))
cols = ['Quarter']
for i in ['Sales', 'Customers']:
for j in cols:
tmp = pd.pivot_table(tmp_data, i, ['Store', 'Year',j], aggfunc='mean' ).reset_index().rename(columns={i: i+'_Year_'+j})
 df = pd.merge(df, tmp, 'left', ['Store','Year', j])
 cols = ['DayOfWeek', 'Promo', 'Promo2', 'Month', 'Year', 'Day', 'StateHoliday', 'Assortment']
for i in ['SalesPerCustomer']:
for j in cols:
tmp = pd.pivot_table(tmp_data, i, ['Store', 'StoreType',j], aggfunc='mean' ).reset_index().rename(columns={i: i+'_StrType_'+j})
 df = pd.merge(df, tmp, 'left', ['Store','StoreType', j])
 return df
output = pd.read_csv(submission_fp, names=['patientId', 'PredictionString'])
 output.head(60 )<drop_column>
df = prepareDf(all_data )
!rm -rf /kaggle/working/Mask_RCNN<import_modules>
df[df['Store']== 696 ].sort_values('Date' )
import math
import os
import shutil
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import glob
import pydicom
import cv2
from sklearn.model_selection import train_test_split
from tqdm import tqdm<define_variables>
df_train = df[df['part']== 'train']
random_stat = 123
np.random.seed(random_stat )<install_modules>
cols_train = ['Store', 'DayOfWeek', 'Date', 'Sales', 'Promo','PromoOpen',
'StateHoliday', 'SchoolHoliday', 'StoreType', 'Assortment',
'CompetitionDistance', 'Promo2', 'Month', 'Year', 'Day','IsPromoMonth',
'Sales_StrType_DayOfWeek','SalesPerCustomer_StrType_Assortment', 'Customers_StrType_Promo2',
'Sales_StrType_Promo', 'Sales_StrType_Promo2', 'Sales_StrType_Month','Customers_StrType_StateHoliday',
'Sales_StrType_Year', 'Sales_StrType_Day', 'Sales_StrType_StateHoliday','SalesPerCustomer_StrType_Promo2',
'Sales_StrType_Assortment', 'Customers_StrType_DayOfWeek', 'Customers_StrType_Promo',
'Customers_StrType_Month', 'Customers_StrType_Year', 'Customers_StrType_Day', 'SalesPerCustomer_StrType_StateHoliday',
'Customers_StrType_Assortment', 'SalesPerCustomer_StrType_DayOfWeek', 'SalesPerCustomer_StrType_Promo',
'SalesPerCustomer_StrType_Month', 'SalesPerCustomer_StrType_Year', 'SalesPerCustomer_StrType_Day',
]
!git clone https://github.com/pjreddie/darknet.git
!cd darknet && sed '1 s/^.*$/GPU=1/; 2 s/^.*$/CUDNN=1/' -i Makefile
!cd darknet && make -j 999 -s
!cp darknet/darknet darknet_gpu<load_from_csv>
params = {"objective": "reg:linear",
"booster" : "gbtree",
"eta": 0.02,
"max_depth": 11,
"subsample": 0.9,
"colsample_bytree": 0.7,
"silent": 1,
"seed": 10,
'tree_method': 'gpu_hist',
}
num_boost_round = 800
def rmspe_xg(yhat, y):
y = np.expm1(y.get_label())
 yhat = np.expm1(yhat)
 return "rmspe", rmspe(y,yhat)
annots = pd.read_csv(os.path.join(DATA_DIR, "stage_1_train_labels.csv"))
annots.head()<train_model>
def process(x, cols=None, all_stores=False):
x.sort_values("Date",inplace=True)
 if cols is None:
cols = x.columns
x = x.fillna(x.median())
 x_train = x[x["Date"]<="2015-06-12"][cols].copy()
x_test = x[x["Date"]>"2015-06-12"][cols].copy()
store_test = x_test['Store'].unique().tolist()
x_train = x_train[(x_train['Store'].isin(store_test)) ]
y_train = np.log(x_train['Sales'])
 if all_stores:
rmv = ['Date', 'Sales']
else:
rmv = ['Date', 'Sales', 'Store']
x_train= x_train.drop(rmv, 1)
 x_train = pd.get_dummies(x_train)
 x_train_arr = x_train.values
x_test_arr = pd.get_dummies(x_test.drop(rmv, 1)).values
return x_train.columns, x_train_arr, y_train, x_test, x_test_arr
def save_img_from_dcm(dcm_dir, img_dir, patient_id):
img_fp = os.path.join(img_dir, "{}.jpg".format(patient_id))
if os.path.exists(img_fp):
return
dcm_fp = os.path.join(dcm_dir, "{}.dcm".format(patient_id))
img_1ch = pydicom.read_file(dcm_fp ).pixel_array
img_3ch = np.stack([img_1ch]*3, -1)
 img_fp = os.path.join(img_dir, "{}.jpg".format(patient_id))
cv2.imwrite(img_fp, img_3ch)
 def save_label_from_dcm(label_dir, patient_id, row=None):
img_size = 1024
label_fp = os.path.join(label_dir, "{}.txt".format(patient_id))
f = open(label_fp, "a")
 if row is None:
f.close()
return
top_left_x = row[1]
top_left_y = row[2]
w = row[3]
h = row[4]
rx = top_left_x/img_size
ry = top_left_y/img_size
rw = w/img_size
rh = h/img_size
rcx = rx+rw/2
rcy = ry+rh/2
line = "{} {} {} {} {}
".format(0, rcx, rcy, rw, rh)
 f.write(line)
 f.close()
def save_yolov3_data_from_rsna(dcm_dir, img_dir, label_dir, annots):
for row in tqdm(annots.values):
patient_id = row[0]
img_fp = os.path.join(img_dir, "{}.jpg".format(patient_id))
if os.path.exists(img_fp):
save_label_from_dcm(label_dir, patient_id, row)
 continue
target = row[5]
if target == 0:
continue
save_label_from_dcm(label_dir, patient_id, row)
 save_img_from_dcm(dcm_dir, img_dir, patient_id )<drop_column>
fig, ax = plt.subplots(5, 2, figsize=(25, 15))
X = df_train[df_train['Store'].isin(top_stores)].copy()
X_train_col, X_train_arr, Y_train, X_test, X_test_arr = process(X, cols_train, True)
 dtrain = xgb.DMatrix(X_train_arr, Y_train)
 estimator = xgb.train(params, dtrain, num_boost_round, feval=rmspe_xg,)
 Y_pred = estimator.predict(xgb.DMatrix(X_test_arr))
X_test["Pred"] = np.exp(Y_pred)
 scores = np.round(mean_squared_error(X_test['Sales'], X_test["Pred"]))
cpt = 0
for i in top_stores[:5]:
x_train = df_train[df_train["Store"]==i]
x_test = X_test[X_test["Store"]==i]
ax[cpt, 0].plot(x_train["Date"], x_train["Sales"])
 ax[cpt, 0].plot(x_test["Date"], x_test["Pred"])
 ax[cpt, 0].set_title(i)
 ax[cpt, 1].scatter(x_test["Date"].values, x_test['Sales'].values - x_test["Pred"].values)
 ax[cpt, 1].plot(x_test["Date"], [0 for _ in range(len(x_test)) ])
 ax[cpt, 1].set_title(np.round(mean_squared_error(X_test['Sales'], X_test["Pred"])))
 cpt+=1
plt.tight_layout()
print(np.mean(scores))
save_yolov3_data_from_rsna(train_dcm_dir, img_dir, label_dir, annots )<prepare_x_and_y>
num_boost_round = 500
params = {"objective": "reg:linear",
"booster" : "gbtree",
"eta": 0.03,
"max_depth": 10,
"subsample": 0.9,
"colsample_bytree": 0.7,
"silent": 1,
"seed": 10,
'tree_method': 'gpu_hist',
}
tmp = prepareDf(all_data, True)
 tmp = tmp.fillna(tmp.dropna().median())
 test = tmp[tmp['part']=='test'].sort_values('Id')
 df_train = tmp[tmp['part']=='train']
result = pd.Series()
X_train = df_train
X_train= X_train[cols_train]
X_test = test
store_test = X_test['Store'].unique().tolist()
X_train = X_train[(X_train['Store'].isin(store_test)) ]
store_ind = X_test["Id"]
Y_train = np.log(X_train["Sales"])
 X_train = pd.get_dummies(X_train.drop(['Date', 'Sales'], 1)).values
X_test = pd.get_dummies(X_test[cols_train] ).drop(['Date', 'Sales'], 1 ).values
dtrain = xgb.DMatrix(X_train, Y_train)
 estimator = xgb.train(params, dtrain, num_boost_round, feval=rmspe_xg,)
 Y_pred = np.exp(estimator.predict(xgb.DMatrix(X_test)))
 result = result.append(pd.Series(Y_pred, index=store_ind))
results = pd.DataFrame({ "Id": result.index, "Sales": result.values} )
ex_patient_id = annots[annots.Target == 1].patientId.values[0]
ex_img_path = os.path.join(img_dir, "{}.jpg".format(ex_patient_id))
ex_label_path = os.path.join(label_dir, "{}.txt".format(ex_patient_id))
plt.imshow(cv2.imread(ex_img_path))
img_size = 1014
with open(ex_label_path, "r")as f:
for line in f:
print(line)
 class_id, rcx, rcy, rw, rh = list(map(float, line.strip().split()))
x =(rcx-rw/2)*img_size
y =(rcy-rh/2)*img_size
w = rw*img_size
h = rh*img_size
plt.plot([x, x, x+w, x+w, x], [y, y+h, y+h, y, y] )<split>
df_test = pd.read_csv('.. /input/test.csv')
 df_test = pd.merge(df_test, stores, 'left', 'Store')
 df_test.head()
<train_model><EOS> 
merged_test = pd.merge(df_test, results, 'left', ['Id'])
 merged_test.loc[ merged_test.Open == 0, 'Sales' ] = 0
sub = merged_test[['Id', 'Sales']].copy()
sub['Sales'] = sub['Sales'].fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub.info()