Kaggle competition: Rossmann Store Sales
<SOS> metric: RootMeanSquarePercentageError Kaggle data source: rossmann-store-sales<train_model>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
tar = tarfile.open('/kaggle/input/external-datasets/rossmann.tgz', "r:gz" )
tr_reg, te_reg, tknzr_reg = tknzr_fit('region', df_x_train, df_test)
 tr_pcn, te_pcn, tknzr_pcn = tknzr_fit('parent_category_name', df_x_train, df_test)
 tr_cn, te_cn, tknzr_cn = tknzr_fit('category_name', df_x_train, df_test)
 tr_ut, te_ut, tknzr_ut = tknzr_fit('user_type', df_x_train, df_test)
 tr_city, te_city, tknzr_city = tknzr_fit('city', df_x_train, df_test)
 tr_p1, te_p1, tknzr_p1 = tknzr_fit('param_1', df_x_train, df_test)
 tr_p2, te_p2, tknzr_p2 = tknzr_fit('param_2', df_x_train, df_test)
 tr_p3, te_p3, tknzr_p3 = tknzr_fit('param_3', df_x_train, df_test )<data_type_conversions>
path = "/kaggle/input/rossmann-store-sales/"
base_path=".. /output"
tr_week = pd.to_datetime(df_x_train['activation_date'] ).dt.weekday.astype(np.int32 ).values
te_week = pd.to_datetime(df_test['activation_date'] ).dt.weekday.astype(np.int32 ).values
tr_week = np.expand_dims(tr_week, axis=-1)
 te_week = np.expand_dims(te_week, axis=-1 )<data_type_conversions>
train = "/kaggle/input/rossmann-store-sales/train.csv"
test = "/kaggle/input/rossmann-store-sales/test.csv"
store = "/kaggle/input/rossmann-store-sales/store.csv"
store_states = tar.extractfile('store_states.csv')
 state_names = tar.extractfile('state_names.csv')
 googletrend = tar.extractfile('googletrend.csv')
 weather = tar.extractfile('weather.csv' )
tr_imgt1 = df_x_train['image_top_1'].astype(np.int32 ).values
te_imgt1 = df_test['image_top_1'].astype(np.int32 ).values
tr_imgt1 = np.expand_dims(tr_imgt1, axis=-1)
 te_imgt1 = np.expand_dims(te_imgt1, axis=-1 )<normalization>
table_names = [train, store, store_states, state_names, googletrend, weather, test]
tables = [pd.read_csv(fpath, low_memory=False)for fpath in table_names]
train, store, store_states, state_names, googletrend, weather, test = tables
len(train),len(test )
eps = 1e-10
tr_price = np.log(df_x_train['price']+eps)
 te_price = np.log(df_test['price']+eps)
 tr_price[tr_price.isna() ] = -1.
 te_price[te_price.isna() ] = -1.
 tr_price = np.expand_dims(tr_price, axis=-1)
 te_price = np.expand_dims(te_price, axis=-1 )<feature_engineering>
train.StateHoliday = train.StateHoliday!='0'
test.StateHoliday = test.StateHoliday!='0'
tr_itemseq = np.log(df_x_train['item_seq_number'])
 te_itemseq = np.log(df_test['item_seq_number'])
 tr_itemseq = np.expand_dims(tr_itemseq, axis=-1)
 te_itemseq = np.expand_dims(te_itemseq, axis=-1 )<set_options>
def join_df(left, right, left_on, right_on=None, suffix='_y'):
if right_on is None: right_on = left_on
return left.merge(right, how='left', left_on=left_on, right_on=right_on,
suffixes=("",suffix))
config.len_desc = 100000<import_modules>
weather = join_df(weather, state_names, "file", "StateName")
 weather.head(3 )
from keras.preprocessing.sequence import pad_sequences<choose_model_class>
googletrend['Date'] = googletrend.week.str.split(' - ', expand=True)[0]
googletrend['State'] = googletrend.file.str.split('_', expand=True)[2]
googletrend.loc[googletrend.State=='NI', "State"] = 'HB,NI'
tknzr_desc = Tokenizer(num_words=config.len_desc, lower='True' )<train_model>
def add_datepart(df, fldname, drop=True, time=False):
"Helper function that adds columns relevant to a date."
fld = df[fldname]
fld_dtype = fld.dtype
if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):
fld_dtype = np.datetime64
if not np.issubdtype(fld_dtype, np.datetime64):
df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)
 targ_pre = re.sub('[Dd]ate$', '', fldname)
 attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',
'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']
if time: attr = attr + ['Hour', 'Minute', 'Second']
for n in attr: df[targ_pre + n] = getattr(fld.dt, n.lower())
 df[targ_pre + 'Elapsed'] = fld.astype(np.int64)// 10 ** 9
if drop: df.drop(fldname, axis=1, inplace=True )
tknzr_desc.fit_on_texts(df_x_train['description'].values )<categorify>
add_datepart(googletrend,"Date", drop=False)
 googletrend.head(3 )
tr_desc_seq = tknzr_desc.texts_to_sequences(df_x_train['description'].values)
 te_desc_seq = tknzr_desc.texts_to_sequences(df_test['description'].values )<set_options>
add_datepart(weather, "Date", drop=False)
 add_datepart(train, "Date", drop=False)
 add_datepart(test, "Date", drop=False )
config.maxlen= 75<categorify>
trend_de = googletrend[googletrend.file == 'Rossmann_DE']
trend_de.head(3 )
tr_desc_pad = pad_sequences(tr_desc_seq, maxlen=config.maxlen)
 te_desc_pad = pad_sequences(te_desc_seq, maxlen=config.maxlen )<set_options>
store = join_df(store, store_states, "Store")
 len(store[store.State.isnull() ] )
gc.collect()<set_options>
joined = join_df(train, store, "Store")
 joined_test = join_df(test, store, "Store")
 len(joined[joined.StoreType.isnull() ]), len(joined_test[joined_test.StoreType.isnull() ] )
config.len_reg = len(tknzr_reg.word_index)
 config.len_pcn = len(tknzr_pcn.word_index)
 config.len_cn = len(tknzr_cn.word_index)
 config.len_ut = len(tknzr_ut.word_index)
 config.len_city = len(tknzr_city.word_index)+1
config.len_week = 7
config.len_imgt1 = int(df_x_train['image_top_1'].max())+1
config.len_p1 = len(tknzr_p1.word_index)+1
config.len_p2 = len(tknzr_p2.word_index)+1
config.len_p3 = len(tknzr_p3.word_index)+1
config.len_price = 1
config.len_itemseq = 1 
<set_options>
joined = join_df(joined, googletrend, ["State","Year", "Week"])
 joined_test = join_df(joined_test, googletrend, ["State","Year", "Week"])
 len(joined[joined.trend.isnull() ]),len(joined_test[joined_test.trend.isnull() ] )
config.emb_reg = 8
config.emb_pcn = 4
config.emb_cn = 8
config.emb_ut = 2
config.emb_city = 16
config.emb_week = 4
config.emb_imgt1 = 16
config.emb_p1 = 8
config.emb_p2 = 16
config.emb_p3 = 16
config.emb_price = 16
config.emb_itemseq = 16
config.emb_desc = 100<prepare_x_and_y>
joined = joined.merge(trend_de, 'left', ["Year", "Week"], suffixes=('', '_DE'))
joined_test = joined_test.merge(trend_de, 'left', ["Year", "Week"], suffixes=('', '_DE'))
len(joined[joined.trend_DE.isnull() ]),len(joined_test[joined_test.trend_DE.isnull() ] )
X = np.array([tr_reg, tr_pcn, tr_cn, tr_ut, tr_city, tr_week, tr_imgt1, tr_p1, tr_p2, tr_p3,
tr_price, tr_itemseq])
 X_test = np.array([te_reg, te_pcn, te_cn, te_ut, te_city, te_week, te_imgt1, te_p1, te_p2, te_p3,
te_price, te_itemseq])
 Y = df_y_train<drop_column>
joined = join_df(joined, weather, ["State","Date"])
 joined_test = join_df(joined_test, weather, ["State","Date"])
 len(joined[joined.Mean_TemperatureC.isnull() ]),len(joined_test[joined_test.Mean_TemperatureC.isnull() ] )
X_train = [x[train_idx] for x in X]
X_valid = [x[valid_idx] for x in X]
X_test = [x for x in X_test]
Y_train = Y[train_idx]
Y_valid = Y[valid_idx]<concatenate>
for df in(joined, joined_test):
for c in df.columns:
if c.endswith('_y'):
if c in df.columns: df.drop(c, inplace=True, axis=1 )
X_train.append(tr_desc_pad[train_idx])
 X_valid.append(tr_desc_pad[valid_idx])
 X_test.append(te_desc_pad )<set_options>
for df in(joined,joined_test):
df['CompetitionOpenSinceYear'] = df.CompetitionOpenSinceYear.fillna(1900 ).astype(np.int32)
 df['CompetitionOpenSinceMonth'] = df.CompetitionOpenSinceMonth.fillna(1 ).astype(np.int32)
 df['Promo2SinceYear'] = df.Promo2SinceYear.fillna(1900 ).astype(np.int32)
 df['Promo2SinceWeek'] = df.Promo2SinceWeek.fillna(1 ).astype(np.int32 )
gc.collect()<compute_test_metric>
for df in(joined, joined_test):
df['CompetitionOpenSince'] = pd.to_datetime(dict(year=df.CompetitionOpenSinceYear,
month=df.CompetitionOpenSinceMonth,
day=15))
df['CompetitionDaysOpen'] = df.Date.subtract(df.CompetitionOpenSince ).dt.days
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)) )<define_variables>
for df in(joined, joined_test):
df.loc[df.CompetitionDaysOpen<0, "CompetitionDaysOpen"] = 0
df.loc[df.CompetitionOpenSinceYear<1990, "CompetitionDaysOpen"] = 0
config.batch_size = 4096<choose_model_class>
for df in(joined,joined_test):
df["CompetitionMonthsOpen"] = df["CompetitionDaysOpen"]//30
df.loc[df.CompetitionMonthsOpen>24, "CompetitionMonthsOpen"] = 24
joined.CompetitionMonthsOpen.unique()
def get_model() :
K.clear_session()
inp_reg = Input(shape=(1,), name='inp_region')
 emb_reg = Embedding(config.len_reg, config.emb_reg, name='emb_region' )(inp_reg)
 inp_pcn = Input(shape=(1,), name='inp_parent_category_name')
 emb_pcn = Embedding(config.len_pcn, config.emb_pcn, name='emb_parent_category_name' )(inp_pcn)
 inp_cn = Input(shape=(1,), name='inp_category_name')
 emb_cn = Embedding(config.len_cn, config.emb_cn, name="emb_category_name" )(inp_cn)
 inp_ut = Input(shape=(1,), name='inp_user_type')
 emb_ut = Embedding(config.len_ut, config.emb_ut, name='emb_user_type' )(inp_ut)
 inp_city = Input(shape=(1,), name='inp_city')
 emb_city = Embedding(config.len_city, config.emb_city, name='emb_city' )(inp_city)
 inp_week = Input(shape=(1,), name='inp_week')
 emb_week = Embedding(config.len_week, config.emb_week, name='emb_week' )(inp_week)
 inp_imgt1 = Input(shape=(1,), name='inp_imgt1')
 emb_imgt1 = Embedding(config.len_imgt1, config.emb_imgt1, name='emb_imgt1' )(inp_imgt1)
 inp_p1 = Input(shape=(1,), name='inp_p1')
 emb_p1 = Embedding(config.len_p1, config.emb_p1, name='emb_p1' )(inp_p1)
 inp_p2 = Input(shape=(1,), name='inp_p2')
 emb_p2 = Embedding(config.len_p2, config.emb_p2, name='emb_p2' )(inp_p2)
 inp_p3 = Input(shape=(1,), name='inp_p3')
 emb_p3 = Embedding(config.len_p3, config.emb_p3, name='emb_p3' )(inp_p3)
 conc_cate = concatenate([emb_reg, emb_pcn, emb_cn, emb_ut, emb_city, emb_week, emb_imgt1, emb_p1, emb_p2, emb_p3], axis=-1, name='concat_categorcal_vars')
 conc_cate = GlobalMaxPool1D()(conc_cate)
 inp_price = Input(shape=(1,), name='inp_price')
 emb_price = Dense(config.emb_price, activation='tanh', name='emb_price' )(inp_price)
 inp_itemseq = Input(shape=(1,), name='inp_itemseq')
 emb_itemseq = Dense(config.emb_itemseq, activation='tanh', name='emb_itemseq' )(inp_itemseq)
 conc_cont = concatenate([conc_cate, emb_price, emb_itemseq], axis=-1)
 x = Dense(200, activation='relu' )(conc_cont)
 x = Dense(50, activation='relu' )(x)
 inp_desc = Input(shape=(config.maxlen,), name='inp_desc')
 emb_desc = Embedding(config.len_desc, config.emb_desc, name='emb_desc' )(inp_desc)
 desc_layer = GRU(40, return_sequences=False )(emb_desc)
 conc_desc = concatenate([x, desc_layer], axis=-1)
 outp = Dense(1, activation='sigmoid', name='output' )(conc_desc)
 model = Model(inputs = [inp_reg, inp_pcn, inp_cn, inp_ut, inp_city, inp_week, inp_imgt1, inp_p1, inp_p2, inp_p3,
inp_price, inp_itemseq, inp_desc], outputs = outp)
 return model<choose_model_class>
for df in(joined, joined_test):
df["Promo2Since"] = pd.to_datetime(df.apply(
 lambda x: Week(x.Promo2SinceYear, x.Promo2SinceWeek ).monday() , axis=1))
df["Promo2Days"] = df.Date.subtract(df['Promo2Since'] ).dt.days
model = get_model()
model.compile(optimizer='adam', loss = root_mean_squared_error, metrics=[root_mean_squared_error])
 model.summary()<choose_model_class>
for df in(joined,joined_test):
df.loc[df.Promo2Days<0, "Promo2Days"] = 0
df.loc[df.Promo2SinceYear<1990, "Promo2Days"] = 0
df["Promo2Weeks"] = df["Promo2Days"]//7
df.loc[df.Promo2Weeks<0, "Promo2Weeks"] = 0
df.loc[df.Promo2Weeks>25, "Promo2Weeks"] = 25
df.Promo2Weeks.unique()
checkpoint = ModelCheckpoint('best.hdf5', monitor='val_loss', verbose=1, save_best_only=True)
 early = EarlyStopping(patience=3, mode='min' )<train_model>
def get_elapsed(fld, pre):
day1 = np.timedelta64(1, 'D')
 last_date = np.datetime64()
last_store = 0
res = []
for s,v,d in zip(df.Store.values, df[fld].values, df.Date.values):
if s != last_store:
last_date = np.datetime64()
last_store = s
if v: last_date = d
res.append(((d-last_date ).astype('timedelta64[D]')/ day1))
df[pre+fld] = res
model.fit(x=X_train, y=np.array(Y_train), validation_data=(X_valid, Y_valid), batch_size=config.batch_size, epochs=6, callbacks=[checkpoint,early], verbose=1 )<save_to_csv>
columns = ["Date", "Store", "Promo", "StateHoliday", "SchoolHoliday"]
model.load_weights('best.hdf5')
 pred = model.predict(X_test)
 subm = pd.read_csv(".. /input/sample_submission.csv")
 subm['deal_probability'] = pred
subm.to_csv('submit_{}_{:.4f}.csv'.format('nn_p3', 0.226), index=False )<load_from_csv>
df = train[columns].append(test[columns])
 df.head(3 )
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
warnings.simplefilter(action='ignore', category=FutureWarning)
 @contextmanager
def timer(name):
t0 = time.time()
yield
print(f'[{name}] done in {time.time() - t0:.0f} s')
 with timer('reading data'):
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 with timer('imputation'):
train['param_1'].fillna('missing', inplace=True)
 test['param_1'].fillna('missing', inplace=True)
 train['param_2'].fillna('missing', inplace=True)
 test['param_2'].fillna('missing', inplace=True)
 train['param_3'].fillna('missing', inplace=True)
 test['param_3'].fillna('missing', inplace=True)
 train['image_top_1'].fillna(0, inplace=True)
 test['image_top_1'].fillna(0, inplace=True)
 train['price'].fillna(0, inplace=True)
 test['price'].fillna(0, inplace=True)
 train['price'] = np.log1p(train['price'])
 test['price'] = np.log1p(test['price'])
 price_mean = train['price'].mean()
price_std = train['price'].std()
train['price'] =(train['price'] - price_mean)/ price_std
test['price'] =(test['price'] - price_mean)/ price_std
train['description'].fillna('', inplace=True)
 test['description'].fillna('', inplace=True)
 train['city'] = train['city'] + '_' + train['region']
test['city'] = test['city'] + '_' + test['region']
with timer('add new features'):
cat_cols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'user_type']
num_cols = ['price', 'deal_probability']
for c in cat_cols:
for c2 in num_cols:
enc = train.groupby(c)[c2].agg(['mean'] ).astype(np.float32 ).reset_index()
enc.columns = ['_'.join([str(c), str(c2), str(c3)])if c3 != c else c for c3 in enc.columns]
train = pd.merge(train, enc, how='left', on=c)
 test = pd.merge(test, enc, how='left', on=c)
 del(enc)
 train.head()<data_type_conversions>
fld = 'SchoolHoliday'
df = df.sort_values(['Store', 'Date'])
 get_elapsed(fld, 'After')
 df = df.sort_values(['Store', 'Date'], ascending=[True, False])
 get_elapsed(fld, 'Before' )
def preprocess(df: pd.DataFrame)-> pd.DataFrame:
ex_col = ['item_id', 'user_id', 'deal_probability', 'title', 'param_1', 'param_2', 'param_3', 'activation_date']
df['description_len'] = df['description'].map(lambda x: len(str(x)) ).astype(np.float16)
 df['description_wc'] = df['description'].map(lambda x: len(str(x ).split(' ')) ).astype(np.float16)
 df['description'] =(df['parent_category_name'] + ' ' + df['category_name'] + ' ' + df['param_1'] + ' ' + df['param_2'] + ' ' + df['param_3'] + ' ' +
df['title'] + ' ' + df['description'].fillna(''))
df['description'] = df['description'].str.lower().replace(r"[^[:alpha:]]", " ")
 df['description'] = df['description'].str.replace(r"\\s+", " ")
 df['title_len'] = df['title'].map(lambda x: len(str(x)) ).astype(np.float16)
 df['title_wc'] = df['title'].map(lambda x: len(str(x ).split(' ')) ).astype(np.float16)
 df['image'] = df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)
 df['price'] = np.log1p(df['price'].fillna(0))
df['wday'] = pd.to_datetime(df['activation_date'] ).dt.dayofweek
col = [c for c in df.columns if c not in ex_col]
return df[col]
with timer('process train'):
train, valid = train_test_split(train, test_size=0.05, shuffle=True, random_state=37)
 y_train = train['deal_probability'].values
X_train = preprocess(train)
 print(f'X_train: {X_train.shape}')
 with timer('process valid'):
X_valid = preprocess(valid)
 print(f'X_valid: {X_valid.shape}')
 with timer('process test'):
X_test = preprocess(test)
 print(f'X_test: {X_test.shape}')
 X_train.head()<feature_engineering>
fld = 'StateHoliday'
df = df.sort_values(['Store', 'Date'])
 get_elapsed(fld, 'After')
 df = df.sort_values(['Store', 'Date'], ascending=[True, False])
 get_elapsed(fld, 'Before' )
desc_len_mean = X_train['description_len'].mean()
desc_len_std = X_train['description_len'].std()
X_train['description_len'] =(X_train['description_len'] - desc_len_mean)/ desc_len_std
X_valid['description_len'] =(X_valid['description_len'] - desc_len_mean)/ desc_len_std
X_test['description_len'] =(X_test['description_len'] - desc_len_mean)/ desc_len_std
desc_wc_mean = X_train['description_wc'].mean()
desc_wc_std = X_train['description_wc'].std()
X_train['description_wc'] =(X_train['description_wc'] - desc_wc_mean)/ desc_wc_std
X_valid['description_wc'] =(X_valid['description_wc'] - desc_wc_mean)/ desc_wc_std
X_test['description_wc'] =(X_test['description_wc'] - desc_wc_mean)/ desc_wc_std
title_len_mean = X_train['title_len'].mean()
title_len_std = X_train['title_len'].std()
X_train['title_len'] =(X_train['title_len'] - title_len_mean)/ title_len_std
X_valid['title_len'] =(X_valid['title_len'] - title_len_mean)/ title_len_std
X_test['title_len'] =(X_test['title_len'] - title_len_mean)/ title_len_std
title_wc_mean = X_train['title_wc'].mean()
title_wc_std = X_train['title_wc'].std()
X_train['title_wc'] =(X_train['title_wc'] - title_wc_mean)/ title_wc_std
X_valid['title_wc'] =(X_valid['title_wc'] - title_wc_mean)/ title_wc_std
X_test['title_wc'] =(X_test['title_wc'] - title_wc_mean)/ title_wc_std
image_top_1_mean = X_train['image_top_1'].mean()
image_top_1_std = X_train['image_top_1'].std()
X_train['image_top_1'] =(X_train['image_top_1'] - image_top_1_mean)/ image_top_1_std
X_valid['image_top_1'] =(X_valid['image_top_1'] - image_top_1_mean)/ image_top_1_std
X_test['image_top_1'] =(X_test['image_top_1'] - image_top_1_mean)/ image_top_1_std<train_model>
fld = 'Promo'
df = df.sort_values(['Store', 'Date'])
 get_elapsed(fld, 'After')
 df = df.sort_values(['Store', 'Date'], ascending=[True, False])
 get_elapsed(fld, 'Before' )
X_train.fillna(0, inplace=True)
 X_valid.fillna(0, inplace=True)
 X_test.fillna(0, inplace=True )<categorify>
df = df.set_index('Date' )
with timer('TFIDF'):
tfidf = TfidfVectorizer(ngram_range=(1, 2),
max_features=100000,
token_pattern='\w+',
encoding='KOI8-R')
 tfidf_train = tfidf.fit_transform(X_train['description'])
 tfidf_valid = tfidf.transform(X_valid['description'])
 tfidf_test = tfidf.transform(X_test['description'] )<categorify>
columns = ['SchoolHoliday', 'StateHoliday', 'Promo']
with timer('Dummy'):
dummy_cols = ['parent_category_name', 'category_name', 'user_type', 'image_top_1', 'wday', 'region', 'city']
for col in dummy_cols:
le = LabelEncoder()
le.fit(X_train[col] + X_valid[col] + X_test[col])
 le.fit(list(X_train[col].values.astype('str')) + list(X_valid[col].values.astype('str')) + list(X_test[col].values.astype('str')))
 X_train[col] = le.transform(list(X_train[col].values.astype('str')))
 X_valid[col] = le.transform(list(X_valid[col].values.astype('str')))
 X_test[col] = le.transform(list(X_test[col].values.astype('str')))
 with timer('Dropping'):
X_train.drop('description', axis=1, inplace=True)
 X_valid.drop('description', axis=1, inplace=True)
 X_test.drop('description', axis=1, inplace=True)
 with timer('OHE'):
ohe = OneHotEncoder(categorical_features=[X_train.columns.get_loc(c)for c in dummy_cols])
 X_train = ohe.fit_transform(X_train)
 print(f'X_train: {X_train.shape}')
 X_valid = ohe.transform(X_valid)
 print(f'X_valid: {X_valid.shape}')
 X_test = ohe.transform(X_test)
 print(f'X_test: {X_test.shape}' )<compute_train_metric>
for o in ['Before', 'After']:
for p in columns:
a = o+p
df[a] = df[a].fillna(0 ).astype(int )
def huber_loss(y_true, y_pred, clip_delta=1.0):
error = y_true - y_pred
cond = tf.keras.backend.abs(error)< clip_delta
squared_loss = 0.5 * tf.keras.backend.square(error)
 linear_loss = clip_delta *(tf.keras.backend.abs(error)- 0.5 * clip_delta)
 return tf.where(cond, squared_loss, linear_loss)
 def fit_predict(xs, y_train, loss_fn='mean_squared_error')-> np.ndarray:
X_train, X_test = xs
config = tf.ConfigProto(
 intra_op_parallelism_threads=4, use_per_session_threads=4, inter_op_parallelism_threads=4)
 with tf.Session(graph=tf.Graph() , config=config)as sess, timer('fit_predict'):
ks.backend.set_session(sess)
 model_in = ks.Input(shape=(X_train.shape[1],), dtype='float32', sparse=True)
 out = ks.layers.Dense(192, activation='relu' )(model_in)
 out = ks.layers.Dense(64, activation='relu' )(out)
 out = ks.layers.Dense(64, activation='relu' )(out)
 out = ks.layers.Dense(1 )(out)
 model = ks.Model(model_in, out)
 model.compile(loss=loss_fn, optimizer=ks.optimizers.Adam(lr=2e-3))
for i in range(3):
with timer(f'epoch {i + 1}'):
model.fit(x=X_train, y=y_train, batch_size=2**(8 + i), epochs=1, verbose=0)
 return model.predict(X_test, batch_size=2**(8 + i)) [:, 0]
X_train = X_train.tocsr()
X_valid = X_valid.tocsr()
X_test = X_test.tocsr()
X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_train, y_train, test_size = 0.5, shuffle = False)
 preds_oofs = []
preds_valids = []
preds_tests = []
for r in range(8):
with timer('Round {}'.format(r)) :
if r % 2 == 0:
loss_name = 'huber_loss'
loss = huber_loss
else:
loss_name = 'mean_squared_error'
loss = 'mean_squared_error'
if r >= 4:
print('Running loss = {}, binary = True'.format(loss_name))
xs = [x.astype(np.bool ).astype(np.float32)for x in [X_train_1, X_train_2]]
y_pred1 = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 xs = [x.astype(np.bool ).astype(np.float32)for x in [X_train_2, X_train_1]]
y_pred2 = fit_predict(xs, y_train=y_train_2, loss_fn=loss)
 xs = [x.astype(np.bool ).astype(np.float32)for x in [X_train_1, X_valid]]
y_predf = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 xs = [x.astype(np.bool ).astype(np.float32)for x in [X_train_1, X_test]]
y_predt = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 else:
print('Running loss = {}, binary = False'.format(loss_name))
xs = [X_train_1, X_train_2]
y_pred1 = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 xs = [X_train_2, X_train_1]
y_pred2 = fit_predict(xs, y_train=y_train_2, loss_fn=loss)
 xs = [X_train_1, X_valid]
y_predf = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 xs = [X_train_1, X_test]
y_predt = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 preds_oof = np.concatenate(( y_pred2, y_pred1), axis=0)
 preds_valid = y_predf
preds_test = y_predt
print('Round {} OOF RMSE: {:.4f}'.format(r, np.sqrt(mean_squared_error(train['deal_probability'], preds_oof))))
print('Round {} Valid RMSE: {:.4f}'.format(r, np.sqrt(mean_squared_error(valid['deal_probability'], preds_valid))))
preds_oofs.append(preds_oof)
 preds_valids.append(preds_valid)
 preds_tests.append(preds_test )<compute_test_metric>
bwd = df[['Store']+columns].sort_index().groupby("Store" ).rolling(7, min_periods=1 ).sum()
preds_oof = np.mean(preds_oofs, axis=0)
 print('Overall OOF RMSE: {:.4f}'.format(np.sqrt(mean_squared_error(train['deal_probability'], preds_oof))))
preds_valid = np.mean(preds_valids, axis=0)
 print('Overall Valid RMSE: {:.4f}'.format(np.sqrt(mean_squared_error(valid['deal_probability'], preds_valid))))<compute_test_metric>
fwd = df[['Store']+columns].sort_index(ascending=False
).groupby("Store" ).rolling(7, min_periods=1 ).sum()
np.mean(np.corrcoef(preds_oofs), axis=0 )<load_from_csv>
bwd.drop('Store',1,inplace=True)
 bwd.reset_index(inplace=True )
with timer('reading data'):
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 target = train['deal_probability']
with timer('imputation'):
train['param_1'].fillna('missing', inplace=True)
 test['param_1'].fillna('missing', inplace=True)
 train['param_2'].fillna('missing', inplace=True)
 test['param_2'].fillna('missing', inplace=True)
 train['param_3'].fillna('missing', inplace=True)
 test['param_3'].fillna('missing', inplace=True)
 train['price'].fillna(0, inplace=True)
 test['price'].fillna(0, inplace=True)
 train['city'] = train['city'] + '_' + train['region']
test['city'] = test['city'] + '_' + test['region']
with timer('FE'):
trainp = preprocess(train)
 testp = preprocess(test)
 for col in ['param_1', 'param_2', 'param_3']:
trainp[col] = train[col]
testp[col] = test[col]
print(train.shape)
 train.head()<drop_column>
fwd.drop('Store',1,inplace=True)
 fwd.reset_index(inplace=True )
with timer('drop'):
trainp.drop(['description', 'image'], axis=1, inplace=True)
 testp.drop(['description', 'image'], axis=1, inplace=True)
 print(trainp.shape)
 print(testp.shape )<data_type_conversions>
df = df.merge(bwd, 'left', ['Date', 'Store'], suffixes=['', '_bw'])
 df = df.merge(fwd, 'left', ['Date', 'Store'], suffixes=['', '_fw'] )
with timer('To cat'):
trainp['image_top_1'] = trainp['image_top_1'].astype('str' ).fillna('missing')
 testp['image_top_1'] = testp['image_top_1'].astype('str' ).fillna('missing')
 cat_cols = ['region', 'city', 'parent_category_name', 'category_name',
'param_1', 'param_2', 'param_3', 'user_type', 'image_top_1', 'wday']
for col in trainp.columns:
print(col)
 if col in cat_cols:
trainp[col] = trainp[col].astype('category')
 testp[col] = testp[col].astype('category')
 else:
trainp[col] = trainp[col].astype(np.float64)
 testp[col] = testp[col].astype(np.float64 )<split>
df.drop(columns,1,inplace=True )
with timer('Split'):
train, valid, y_train, y_valid = train_test_split(trainp, target, test_size=0.05, shuffle=True, random_state=37)
 test = testp
print(train.shape)
 print(valid.shape)
 print(test.shape )<create_dataframe>
df["Date"] = pd.to_datetime(df.Date )
with timer('Submodels'):
train_models = pd.DataFrame(np.array(preds_oofs ).transpose())
 valid_models = pd.DataFrame(np.array(preds_valids ).transpose())
 test_models = pd.DataFrame(np.array(preds_tests ).transpose())
 train_models.columns = ['nn_' + str(i + 1)for i in range(train_models.shape[1])]
valid_models.columns = ['nn_' + str(i + 1)for i in range(train_models.shape[1])]
test_models.columns = ['nn_' + str(i + 1)for i in range(train_models.shape[1])]
print(train_models.shape)
 print(valid_models.shape)
 print(test_models.shape )<concatenate>
joined = join_df(joined, df, ['Store', 'Date'] )
with timer('Concat'):
print(train.shape)
 X_train = pd.concat([train.reset_index() , train_models.reset_index() ], axis=1)
 print(X_train.shape)
 print('-')
 print(valid.shape)
 X_valid = pd.concat([valid.reset_index() , valid_models.reset_index() ], axis=1)
 print(X_valid.shape)
 print('-')
 print(test.shape)
 X_test = pd.concat([test.reset_index() , test_models.reset_index() ], axis=1)
 print(X_test.shape)
 X_train.head()<drop_column>
joined_test = join_df(joined_test, df, ['Store', 'Date'] )
del trainp
del testp
gc.collect()<create_dataframe>
joined = joined[joined.Sales!=0]
d_train = lgb.Dataset(X_train, label=y_train)
 d_valid = lgb.Dataset(X_valid, label=y_valid)
 watchlist = [d_train, d_valid]
params = {'application': 'regression',
'metric': 'rmse',
'nthread': 3,
'verbosity': -1,
'data_random_seed': 3,
'learning_rate': 0.05,
'num_leaves': 31,
'bagging_fraction': 0.8,
'feature_fraction': 0.2,
'lambda_l1': 3,
'lambda_l2': 3,
'min_data_in_leaf': 40}
model = lgb.train(params,
train_set=d_train,
num_boost_round=1500,
valid_sets=watchlist,
verbose_eval=100)
 pprint(sorted(list(zip(model.feature_importance() , X_train.columns)) , reverse=True))
print('Done' )<save_to_csv>
joined.reset_index(inplace=True)
 joined_test.reset_index(inplace=True )
valid_preds = model.predict(X_valid ).clip(0, 1)
 print('Overall Valid RMSE: {:.4f}'.format(np.sqrt(mean_squared_error(y_valid, valid_preds))))
test_preds = model.predict(X_test ).clip(0, 1)
 submission = pd.read_csv('.. /input/test.csv', usecols=["item_id"])
 submission["deal_probability"] = test_preds
submission.to_csv("submit_boosting_mlp.csv", index=False, float_format="%.2g" )<set_options>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )
color = sns.color_palette()
%matplotlib inline
py.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999<load_from_csv>
train_df = joined
test_df = joined_test
train_df = pd.read_csv(".. /input/train.csv", parse_dates=["activation_date"])
 test_df = pd.read_csv(".. /input/test.csv", parse_dates=["activation_date"])
 print("Train file rows and columns are : ", train_df.shape)
 print("Test file rows and columns are : ", test_df.shape )<define_variables>
n = len(train_df); n
temp_data = StringIO()
region_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, region_df, how="left", on="region" )<merge>
idx = np.random.permutation(range(n)) [:2000]
idx.sort()
small_train_df = train_df.iloc[idx[:1000]]
small_test_df = train_df.iloc[idx[1000:]]
small_cont_vars = ['CompetitionDistance','Mean_Humidity']
small_cat_vars = ['Store','DayOfWeek','PromoInterval']
small_train_df = small_train_df[small_cat_vars + small_cont_vars + ['Sales']]
small_test_df = small_test_df[small_cat_vars + small_cont_vars + ['Sales']]
temp_data = StringIO()
temp_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, temp_df, on="parent_category_name", how="left" )<define_variables>
categorify = Categorify(small_cat_vars, small_cont_vars)
 categorify(small_train_df)
 categorify(small_test_df, test=True )
temp_data = StringIO()
temp_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, temp_df, on="category_name", how="left" )<categorify>
fill_missing = FillMissing(small_cat_vars, small_cont_vars)
 fill_missing(small_train_df)
 fill_missing(small_test_df, test=True )
tfidf_vec = TfidfVectorizer(ngram_range=(1,1))
full_tfidf = tfidf_vec.fit_transform(train_df['title'].values.tolist() + test_df['title'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['title'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['title'].values.tolist())
 n_comp = 3
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_title_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_title_'+str(i+1)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<categorify>
small_train_df[small_train_df['CompetitionDistance_na'] == True]
tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)
 full_tfidf = tfidf_vec.fit_transform(train_df['description'].values.tolist() + test_df['description'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['description'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['description'].values.tolist())
 n_comp = 3
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_desc_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_desc_'+str(i+1)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<categorify>
procs=[FillMissing, Categorify, Normalize]
train_y = train_df["deal_probability"].values
test_id = test_df["item_id"].values
train_df["activation_weekday"] = train_df["activation_date"].dt.weekday
test_df["activation_weekday"] = test_df["activation_date"].dt.weekday
cat_vars = ["region", "city", "parent_category_name", "category_name", "user_type", "param_1", "param_2", "param_3"]
for col in cat_vars:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))
 train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))
 test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))
 cols_to_drop = ["item_id", "user_id", "title", "description", "activation_date", "image"]
train_X = train_df.drop(cols_to_drop + ["region_en", "parent_category_name_en", "category_name_en", "price_new", "deal_probability"], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1 )<create_dataframe>
cat_vars = ['Store', 'DayOfWeek', 'Year', 'Month', 'Day', 'StateHoliday', 'CompetitionMonthsOpen',
'Promo2Weeks', 'StoreType', 'Assortment', 'PromoInterval', 'CompetitionOpenSinceYear', 'Promo2SinceYear',
'State', 'Week', 'Events', 'Promo_fw', 'Promo_bw', 'StateHoliday_fw', 'StateHoliday_bw',
'SchoolHoliday_fw', 'SchoolHoliday_bw']
cont_vars = ['CompetitionDistance', 'Max_TemperatureC', 'Mean_TemperatureC', 'Min_TemperatureC',
'Max_Humidity', 'Mean_Humidity', 'Min_Humidity', 'Max_Wind_SpeedKm_h',
'Mean_Wind_SpeedKm_h', 'CloudCover', 'trend', 'trend_DE',
'AfterStateHoliday', 'BeforeStateHoliday', 'Promo', 'SchoolHoliday']
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 30,
"learning_rate" : 0.1,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.7,
"bagging_frequency" : 5,
"bagging_seed" : 2018,
"verbosity" : -1
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=20, evals_result=evals_result)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 return pred_test_y, model, evals_result<save_to_csv>
dep_var = 'Sales'
df = train_df[cat_vars + cont_vars + [dep_var,'Date']].copy()
dev_X = train_X.iloc[:-200000,:]
val_X = train_X.iloc[-200000:,:]
dev_y = train_y[:-200000]
val_y = train_y[-200000:]
print(dev_X.shape, val_X.shape, test_X.shape)
 pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X)
 pred_test[pred_test>1] = 1
pred_test[pred_test<0] = 0
sub_df = pd.DataFrame({"item_id":test_id})
 sub_df["deal_probability"] = pred_test
sub_df.to_csv("baseline_lgb.csv", index=False )<define_variables>
cut = train_df['Date'][(train_df['Date'] == train_df['Date'][len(test_df)])].index.max()
cut
on_kaggle = True<init_hyperparams>
valid_idx = range(cut )
OTHRESH=0.05
PTHRESH=0.94
AVPTHRESH=0.65
THRESH =.175
MIN_MAX_MRCNN_CONF =.975
MINPROB = 0.69
C1 = 1e6
C2 = 0.06
UNET_MINPROB = 0.28
UNET_MINCONF = 0.35
YOLO_MINCONF = 0.15
MINMIN = 0.20<import_modules>
data =(TabularList.from_df(df, path='.', cat_names=cat_vars, cont_names=cont_vars, procs=procs,)
.split_by_idx(valid_idx)
.label_from_df(cols=dep_var, label_cls=FloatList, log=True)
.add_test(TabularList.from_df(test_df, path=path, cat_names=cat_vars, cont_names=cont_vars)) 
.databunch() )
import numpy as np
import pandas as pd
import os
from sklearn import metrics
from scipy.special import logit,expit
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import mean_squared_error<define_variables>
max_log_y = np.log(np.max(train_df['Sales'])*1.2)
 y_range = torch.tensor([0, max_log_y], device=defaults.device )
RESULTS_LOC = '.. /input'
RAW_DATA_LOC = '.. /input/rsna-pneumonia-detection-challenge'
YOLO_SUB_STEM = 'yolov3-inference-from-multiple-saved-wts-stage-2/submission_yolo'
YOLO_SUB_STEM2 = 'yolov3-inference-from-more-saved-weights-stage-2/submission_yolo'
MORE_YOLO_EPOCHS = [5300, 1500, 6700, 2000] if on_kaggle else []
UNET_RESULTS_PATH = '.. /input/resnet-unet-from-saved-weights-stage-2/submission_resnet34unet_thr0.35.csv'
CHEXNET_OOF_STEM = 'rsna-oof-predictions/predictions_valid_fold_'
CHEXNET_TEST_STEM = 'gs-dense-chexnet-predict-stage-2-from-all-models/test_preds_pth_fold'
SPECIAL_FILE0 = 'gs-dense-chexnet-predict-stage-2-from-all-models/test_preds_pth_fold0_for_combined_folds'
CHEXNET_TEST_FILE0 = SPECIAL_FILE0 if on_kaggle else CHEXNET_TEST_STEM + '0'
ADENSE_TEST_STEM = 'andy-densenet-from-multiple-saved-weights-stage-2/submission_adense_f'
ADEMSE_OOF_STEM = 'rsna-oof-predictions/val_dense_v5_a1_f'
MRCNN_TEST_STEM = 'mrcnn-inference-from-saved-wts-4dig-70-stage-2/submission_mrcnn_v'
MRCNN95_TEST_STEM = 'mrcnn-inference-from-saved-wts-2dig-95-stage-2/submission_mrcnn_v' \
if on_kaggle else MRCNN_TEST_STEM
MRCNN_OUT_STEM = 'temp_mrcnn_v' if on_kaggle else MRCNN_TEST_STEM
MRCNN_OOF_STEM = 'rsna-oof-predictions/val_v'
MRCNN_OOF_OUT_STEM = 'val_v' if on_kaggle else MRCNN_OOF_STEM
MRCNN_TEST_SEP = ''
MRCNN_OOF_SEP = '_'
IN_OOF_SUFFIX = ''
IN_SUFFIX = '.csv'
OUT_SUFFIX = '.csv'
CHEX_CLASS_PRED_PATH = '.. /input/combine-fold-class-predictions-stage-2/test_probs.csv'
KERNEL_OUTPUT_PATH = '.. /input/filter-199-final-with-higher-thresh-stage-2/filter199.csv'<define_variables>
learn = tabular_learner(data, layers=[1000,500], ps=[0.001,0.01], emb_drop=0.04,
y_range=y_range, metrics=exp_rmspe )
N_FOLDS = 5
OUTPUT_LOC = '.'
OOF_INFILE_STEM = MRCNN_OOF_STEM + '9' + MRCNN_OOF_SEP + 'a1' + MRCNN_OOF_SEP + 'f'
vers = '8' if on_kaggle else '9'
TEST_INFILE_STEM = MRCNN_TEST_STEM + vers + MRCNN_TEST_SEP + 'a1' + MRCNN_TEST_SEP + 'f'
OOF_OUTFILE_NAME = MRCNN_OOF_OUT_STEM + '10' + MRCNN_OOF_SEP + 'a1' + \
MRCNN_OOF_SEP + 'allfolds_out.csv'
TEST_OUTFILE_STEM = MRCNN_OUT_STEM + '10' + MRCNN_TEST_SEP + 'a1' + MRCNN_TEST_SEP + 'f_out'
OOF_CLASS_PROBS_OUTFILE = 'phase3_oof_out.csv'
TEST_CLASS_PROBS_OUTFILE_STEM = 'phase3_test_out'
FULL_TEST_CLASS_PROBS_OUTFILE = 'phase3_test_avg_out.csv'<load_from_csv>
learn.model
dfs = []
for a in range(1,3):
for f in range(5):
vers = '8' if a==2 and f==0 else '1'
fn = RESULTS_LOC +'/'+ MRCNN95_TEST_STEM + vers + MRCNN_TEST_SEP + \
'a' + str(a)+ MRCNN_TEST_SEP + 'f' + str(f)+ IN_SUFFIX
dfs.append(pd.read_csv(fn ).set_index('patientId'))
for i,f in enumerate(dfs):
if i:
df = df.join(f.rename(columns={'PredictionString':'pred'+str(i)}))
else:
df = f.rename(columns={'PredictionString':'pred'+str(i)} )<prepare_x_and_y>
learn.fit_one_cycle(5, 1e-3, wd=0.2 )
def non_max_suppression(boxes, probs=None, overlapThresh=OTHRESH):
	
	if len(boxes)== 0:
		return []
	
	
	if boxes.dtype.kind == "i":
		boxes = boxes.astype("float")
 	
	pick = []
	
	x1 = boxes[:, 0]
	y1 = boxes[:, 1]
	x2 = boxes[:, 2]
	y2 = boxes[:, 3]
	
	
	
	area =(x2 - x1 + 1)*(y2 - y1 + 1)
 	idxs = y2
	
	if probs is not None:
		idxs = probs
	
	idxs = np.argsort(idxs)
 	
	while len(idxs)> 0:
		
		
		last = len(idxs)- 1
		i = idxs[last]
		pick.append(i)
 		
		
		
		xx1 = np.maximum(x1[i], x1[idxs[:last]])
 		yy1 = np.maximum(y1[i], y1[idxs[:last]])
 		xx2 = np.minimum(x2[i], x2[idxs[:last]])
 		yy2 = np.minimum(y2[i], y2[idxs[:last]])
 		
		w = np.maximum(0, xx2 - xx1 + 1)
 		h = np.maximum(0, yy2 - yy1 + 1)
 		
		overlap =(w * h)/ area[idxs[:last]]
		
		
		idxs = np.delete(idxs, np.concatenate(( [last],
			np.where(overlap > overlapThresh)[0])))
 	
	return boxes[pick].astype("int"), list(np.array(probs)[pick] )<define_variables>
learn.save('1' )
box_dict = {}
conf_dict = {}
for pat,row in df.iterrows() :
boxes = []
confs = []
maxconfs = []
for pred in row:
maxconf = 0.
 if isinstance(pred, str):
s = pred.split(' ')
 if s[-1]=='':
s.pop()
if s[0]=='':
s.pop(0)
 if(len(s)%5):
print('Bad prediction string.')
 while len(s):
conf = float(s.pop(0))
x = int(round(float(s.pop(0))))
y = int(round(float(s.pop(0))))
w = int(round(float(s.pop(0))))
h = int(round(float(s.pop(0))))
if conf>maxconf:
maxconf = conf
if conf>PTHRESH:
boxes.append([x,y,x+w,y+h])
 confs.append(conf)
 maxconfs.append(maxconf)
 avgconf = sum(maxconfs)/len(maxconfs)
 if len(boxes)and avgconf>AVPTHRESH:
box_dict[pat] = boxes
conf_dict[pat] = confs
len(box_dict), len(conf_dict )<categorify>
learn.load('1');
box_dict_nms = {}
conf_dict_nms = {}
for p in box_dict:
boxes, confs = non_max_suppression(np.array(box_dict[p]), np.array(conf_dict[p]))
box_dict_nms[p] = boxes
conf_dict_nms[p] = confs<prepare_output>
learn.fit_one_cycle(5, 3e-4 )
<load_from_csv><EOS> 
test_preds=learn.get_preds(DatasetType.Test)
 test_df["Sales"]=np.exp(test_preds[0].data ).numpy().T[0]
test_df[["Id","Sales"]]=test_df[["Id","Sales"]].astype("int")
 test_df[["Id","Sales"]].to_csv("rossmann_submission.csv",index=False )