Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<drop_column>
warnings.filterwarnings('always')
 warnings.filterwarnings('ignore')
 style.use('fivethirtyeight')
 sns.set(style='whitegrid',color_codes=True )
del shop_item_pair_num_month
del ax
del all_pairs
del all_existed_pair_month
del all_items
del all_shops
del item_trend
del item_1
del item_2
del item_3
del fig
del ax1
del ax2
del ax3
gc.collect()<merge>
def seed_everything(seed=999):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def get_label(diagnosis):
return ','.join([str(i)for i in range(diagnosis + 1)])
 def get_train_df(seed, num_zeros=4000):
val_preds_id = pd.read_csv('.. /input/bd-peter-and-lex-validation-set/val.csv')['id_code']
df_train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 df_train['is_valid'] = False
df_train.id_code = '.. /input/aptos2019-blindness-detection/train_images/' + df_train.id_code + '.png'
df_train.columns = ['image_path', 'diagnosis', 'is_valid']
extra_training_df = pd.read_csv('.. /input/diabetic-retinopathy-resized/trainLabels.csv')
 extra_training_df['is_valid'] = False
extra_training_df.image = '.. /input/diabetic-retinopathy-resized/resized_train/resized_train/' + extra_training_df.image + '.jpeg'
extra_training_df.columns = ['image_path', 'diagnosis', 'is_valid']
test_labels_15_df = pd.read_csv('.. /input/resized-2015-2019-blindness-detection-images/labels/testLabels15.csv')
 del test_labels_15_df['Usage']
test_labels_15_df.columns = ['image_id', 'diagnosis']
test_labels_15_df['dataset_id'] = 'test_labels_15'
test_labels_15_df['image_path'] = '.. /input/resized-2015-2019-blindness-detection-images/resized test 15/' + test_labels_15_df.image_id + '.jpg'
test_labels_15_df['is_valid'] = True
test_labels_15_df = test_labels_15_df[['image_path', 'diagnosis', 'is_valid']]
df_train = pd.concat([
df_train,
extra_training_df[(extra_training_df.diagnosis == 0)&(extra_training_df.is_valid)],
extra_training_df[(extra_training_df.diagnosis == 0)& ~(extra_training_df.is_valid)].sample(n=num_zeros, random_state=seed),
extra_training_df[extra_training_df.diagnosis == 1],
extra_training_df[extra_training_df.diagnosis == 2],
extra_training_df[extra_training_df.diagnosis == 3],
extra_training_df[extra_training_df.diagnosis == 4],
pd.concat([
test_labels_15_df[test_labels_15_df.diagnosis == 0].sample(n=7900, random_state=420),
test_labels_15_df[test_labels_15_df.diagnosis != 0]
] ).sample(n=10_000, random_state=420),
] ).sample(frac=1, random_state=seed)
 df_train['label'] = df_train.diagnosis.apply(get_label)
 return df_train
def make_or_preds(model_name, learner, model_path, expected_val):
learn.load(model_path);
val_items = learn.data.valid_dl.dataset.items
val_preds, val_y = learn.get_preds(ds_type=DatasetType.Valid)
 metric = cohen_kappa_score(val_y.argmax(1 ).numpy() , get_output_preds(( val_preds > 0.5 ).numpy()), weights='quadratic')
 raw_preds = pd.DataFrame(val_preds.numpy())
 raw_preds.columns = ['x_0', 'x_1', 'x_2', 'x_3', 'x_4']
val_preds_df = pd.concat([
pd.DataFrame({
'id_code': [v.split('/')[-1].split('.')[0] for v in val_items],
'diagnosis': val_y.argmax(1 ).numpy() ,
'preds': get_output_preds(( val_preds > 0.5 ).numpy())
 }),
raw_preds
], axis=1)
 val_preds_df.to_csv(f'{model_name}_val_preds.csv', index=False)
 test_items = learn.data.test_dl.dataset.items
test_preds, __ = learn.get_preds(ds_type=DatasetType.Test)
 raw_test_preds = pd.DataFrame(test_preds.numpy())
 raw_test_preds.columns = ['x_0', 'x_1', 'x_2', 'x_3', 'x_4']
test_preds_df = pd.concat([
pd.DataFrame({
'id_code': [v.split('/')[-1].split('.')[0] for v in test_items],
'preds': get_output_preds(( test_preds > 0.5 ).numpy())
 }),
raw_test_preds
], axis=1)
 test_preds_df.to_csv(f'{model_name}_test_preds.csv', index=False)
 print(f'Val kappa score: {metric}(expected: {expected_val})')
 def avg_tta_score(model_name):
no_flip = pd.read_csv(f'{model_name}_val_preds.csv' ).sort_values('id_code')
 flip = pd.read_csv(f'{model_name}-flip_val_preds.csv' ).sort_values('id_code')
 val_preds_avg = no_flip[['x_0', 'x_1', 'x_2', 'x_3', 'x_4']].values * 0.5 + flip[['x_0', 'x_1', 'x_2', 'x_3', 'x_4']].values * 0.5
return cohen_kappa_score(flip.diagnosis, get_output_preds(( val_preds_avg > 0.5)) , weights='quadratic')
 class ConfusionMatrix(Callback):
"Computes the confusion matrix."
def on_train_begin(self, **kwargs):
self.n_classes = 0
def on_epoch_begin(self, **kwargs):
self.cm = None
def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):
preds = torch.tensor(get_preds(( torch.sigmoid(last_output)> 0.5 ).cpu().numpy()))
targs = torch.tensor(get_preds(last_target.cpu().numpy()))
if self.n_classes == 0:
self.n_classes = last_output.shape[-1]
self.x = torch.arange(0, self.n_classes)
 cm =(( preds==self.x[:, None])&(targs==self.x[:, None, None])).sum(dim=2, dtype=torch.float32)
 if self.cm is None: self.cm = cm
else: self.cm += cm
def on_epoch_end(self, **kwargs):
self.metric = self.cm
@dataclass
class KappaScore(ConfusionMatrix):
"Compute the rate of agreement(Cohens Kappa )."
weights:Optional[str]=None
def on_epoch_end(self, last_metrics, **kwargs):
sum0 = self.cm.sum(dim=0)
 sum1 = self.cm.sum(dim=1)
 expected = torch.einsum('i,j->ij',(sum0, sum1)) / sum0.sum()
if self.weights is None:
w = torch.ones(( self.n_classes, self.n_classes))
w[self.x, self.x] = 0
elif self.weights == "linear" or self.weights == "quadratic":
w = torch.zeros(( self.n_classes, self.n_classes))
w += torch.arange(self.n_classes, dtype=torch.float)
 w = torch.abs(w - torch.t(w)) if self.weights == "linear" else(w - torch.t(w)) ** 2
else: raise ValueError('Unknown weights.Expected None, "linear", or "quadratic".')
 k = torch.sum(w * self.cm)/ torch.sum(w * expected)
 return add_metrics(last_metrics, 1-k)
 class FlattenedLoss() :
"Same as `func`, but flattens input and target."
def __init__(self, func, *args, axis:int=-1, floatify:bool=False, is_2d:bool=True, **kwargs):
self.func,self.axis,self.floatify,self.is_2d = func(*args,**kwargs),axis,floatify,is_2d
functools.update_wrapper(self, self.func)
 def __repr__(self): return f"FlattenedLoss of {self.func}"
@property
def reduction(self): return self.func.reduction
@reduction.setter
def reduction(self, v): self.func.reduction = v
def __call__(self, input:Tensor, target:Tensor, **kwargs)->Rank0Tensor:
input = input.transpose(self.axis,-1 ).contiguous()
target = target.transpose(self.axis,-1 ).contiguous()
if self.floatify: target = target.float()
target =(target * 0.9 + 0.05)
 target[:,0] = 1
input = input.view(-1,input.shape[-1])if self.is_2d else input.view(-1)
 return self.func.__call__(input, target.view(-1), **kwargs)
 def LabelSmoothBCEWithLogitsFlat(*args, axis:int=-1, floatify:bool=True, **kwargs):
"Same as `nn.BCEWithLogitsLoss`, but flattens input and target."
return FlattenedLoss(nn.BCEWithLogitsLoss, *args, axis=axis, floatify=floatify, is_2d=False, **kwargs)
 class ReconstructFixMultiCategoryList(MultiCategoryList):
def reconstruct(self, t):
try:
return super().reconstruct(t)
 except Exception as e:
return FloatItem(np.log(t))
cnt_month = df.groupby(['shop_id','item_id','date_block_num'] ).sum() ['item_cnt_day']
cnt_month.name = 'item_cnt_month'
dataset = dataset.merge(cnt_month,on=['shop_id','item_id','date_block_num'],how='left')
 dataset['item_cnt_month'] = dataset['item_cnt_month'].fillna(0 ).clip(0,20)
 del cnt_month
gc.collect()<concatenate>
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv' )
df_test['date_block_num'] = 34
dataset = pd.concat([dataset,df_test.drop('ID',axis=1)],axis=0 ).reset_index(drop=True)
 assert dataset.isnull().sum().max() == len(df_test)
 dataset = dataset.fillna(0 )<data_type_conversions>
model = get_efficientnet('b3', True, '.. /input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth' )
float32_cols = ['revenue','avg_price']
int16_cols =['item_id']
int8_cols = ['date_block_num','shop_id','item_cnt_month']
dataset[int8_cols] = dataset[int8_cols].astype('int8')
 dataset[int16_cols] = dataset[int16_cols].astype('int16')
 dataset[float32_cols] = dataset[float32_cols].astype('float32' )<categorify>
seed_everything(422)
 data = get_data(seed=437, size=256, bs=64)
 learn = Learner(data, model, model_dir=".", callback_fns=[BnFreeze])
 make_or_preds(
 model_name='bd-efficientnet-b03-2015val-psu3-sz-256',
learner=learn,
model_path='.. /input/bd-efficientnet-b03-2015val-psu3-sz-256/best_model',
expected_val=0.7273654273342949
)
def add_lag_v3(dataset,lookup_set,lag_months,lag_features_list,on_key,fillna_value,dtypes):

 if lookup_set is None:
lookup_set = dataset[on_key + lag_features_list].copy()
else:
dataset = pd.merge(dataset,lookup_set,how='left',on=on_key)
 for lag in lag_months:
dataset_alignment = dataset[on_key + lag_features_list].copy()
dataset_lag = lookup_set[on_key + lag_features_list]
dataset_lag['date_block_num'] = dataset_lag['date_block_num'] + lag
dataset_alignment = pd.merge(dataset_alignment,dataset_lag,how='left',on=on_key,suffixes=['','_lag_'+str(lag)+'m'])
 dataset_alignment = dataset_alignment.drop(on_key + lag_features_list,axis=1)
 dataset = pd.concat([dataset,dataset_alignment],axis=1,join='outer')
 del dataset_lag
del dataset_alignment
gc.collect()
for i,feature in enumerate(lag_features_list):
new_columns = dataset.columns.str.startswith(feature + '_lag_')
 dataset.loc[:,new_columns] = dataset.loc[:,new_columns].fillna(fillna_value[i] ).astype(dtypes[i])
 return dataset<categorify>
seed_everything(422)
 data = get_data(seed=437, size=256, bs=64, tfms=((),(flip_lr(p=1))))
learn = Learner(data, model, model_dir=".", callback_fns=[BnFreeze])
 make_or_preds(
 model_name='bd-efficientnet-b03-2015val-psu3-sz-256-flip',
learner=learn,
model_path='.. /input/bd-efficientnet-b03-2015val-psu3-sz-256/best_model',
expected_val=0.7273654273342949
)
def add_first_last_sale(dataset,source_set,keys,prefix):

 for m in range(1,35):
tmp = source_set[source_set['date_block_num']<m].drop_duplicates(keys+['date_block_num'] ).groupby(keys ).agg({'date_block_num':[np.min,np.max]})
 tmp.columns = [prefix + '_first_sold', prefix + '_last_sold']
tmp = tmp.reset_index()
tmp['date_block_num'] = m
if m == 1:
first_last_sold = tmp
else:
first_last_sold = pd.concat([first_last_sold,tmp],axis=0)
 del tmp
gc.collect()
dataset = dataset.merge(first_last_sold,on=keys+['date_block_num'],how='left')
 dataset[prefix + '_first_sold_offset'] = dataset[prefix + '_first_sold'] - dataset['date_block_num']
dataset[prefix + '_last_sold_offset'] = dataset[prefix + '_last_sold'] - dataset['date_block_num']
columns = [prefix + '_first_sold', prefix + '_last_sold',prefix + '_first_sold_offset', prefix + '_last_sold_offset']
dataset[columns] = dataset[columns].fillna(0)
 dataset[columns] = dataset[columns].astype('int8')
 return dataset<categorify>
avg_tta_score('bd-efficientnet-b03-2015val-psu3-sz-256' )
def create_mean_encoding_table(dataset:pd.DataFrame, encode_item:list, prefix:str):
encoding_table = dataset[encode_item +['date_block_num','item_cnt_month','revenue','avg_price']]
encoding_table.loc[:,['item_cnt_month','revenue','avg_price']] = encoding_table.loc[:,['item_cnt_month','revenue','avg_price']].replace(0,np.nan)
 encoding_table = encoding_table.groupby(encode_item + ['date_block_num'] ).agg({'item_cnt_month':[np.mean],'revenue':[np.mean],'avg_price':[np.mean]})
 encoding_table = encoding_table.fillna(0)
 encoding_table.columns = [prefix+'_avg_cnt_month', prefix+'_avg_revenue_month',prefix+'_avg_price_month']
encoding_table = encoding_table.reset_index()
encoding_table[['date_block_num',prefix+'_avg_cnt_month']] = encoding_table[['date_block_num',prefix+'_avg_cnt_month']].astype('float16')
 encoding_table[encode_item] = encoding_table[encode_item].astype('int16')
 encoding_table[[prefix+'_avg_revenue_month',prefix+'_avg_price_month']] = encoding_table[[prefix+'_avg_revenue_month',prefix+'_avg_price_month']].astype('float32')
 return encoding_table<create_dataframe>
seed_everything(420)
 data = get_data(seed=420, size=300, bs=64)
 learn = Learner(data, model, model_dir=".", callback_fns=[BnFreeze])
 make_or_preds(
 model_name='bd-efficientnet-b03-2015val-psu3-sz-300',
learner=learn,
model_path='.. /input/bd-efficientnet-b03-2015val-psu3-sz-300/best_model',
expected_val=0.7433966054032339
)
def first_sale_cnt(dataset,source_set,keys,prefix,forward_range=6):

 source_set = source_set[keys+['date_block_num','item_first_sold_offset','item_cnt_month']]
source_set = source_set[source_set['item_first_sold_offset']==0]
source_set = source_set.drop('item_first_sold_offset',axis=1)
 source_set = source_set[source_set['item_cnt_month']>0]
for m in range(1,35):
tmp = source_set[source_set['date_block_num'].between(m-1-6,m-1)].groupby(keys ).agg({'item_cnt_month':[np.mean]})
 tmp.columns = [prefix + '_avg_first_sold_cnt_month']
tmp = tmp.reset_index()
tmp['date_block_num'] = m
if m == 1:
avg_first_sold_cnt = tmp
else:
avg_first_sold_cnt = pd.concat([avg_first_sold_cnt,tmp],axis=0)
 del tmp
dataset = dataset.merge(avg_first_sold_cnt,on=keys+['date_block_num'],how='left')
 columns = [prefix + '_avg_first_sold_cnt_month']
dataset[columns] = dataset[columns].fillna(0)
 dataset[columns] = dataset[columns].astype('float16')
 return dataset<load_from_csv>
seed_everything(420)
 data = get_data(seed=420, size=300, bs=64, tfms=((),(flip_lr(p=1))))
learn = Learner(data, model, model_dir=".", callback_fns=[BnFreeze])
 make_or_preds(
 model_name='bd-efficientnet-b03-2015val-psu3-sz-300-flip',
learner=learn,
model_path='.. /input/bd-efficientnet-b03-2015val-psu3-sz-300/best_model',
expected_val=0.7433966054032339
)
def get_google_trends(IPs, file_path, reference):

 pytrend = TrendReq(hl='ru', tz=360)
 try:
IP_trend = pd.read_csv(file_path)
 existed_columns = IP_trend.columns
except FileNotFoundError:
IP_trend = None
existed_columns = []
for IP in IPs:
if IP in existed_columns:
continue
if IP == reference:
keywords = [reference]
else:
keywords = [IP,reference]
pytrend.build_payload(kw_list=keywords,cat=0,timeframe='2013-01-01 2015-10-31',geo='RU',gprop='')
 trend_df = pytrend.interest_over_time()
trend_df = trend_df.drop(reference,axis=1)
 if len(trend_df)== 0:
if IP_trend is not None:
IP_trend[IP] = 0
else:
trend_df = trend_df.reset_index()
trend_df['year'] = trend_df['date'].dt.year
trend_df['month'] = trend_df['date'].dt.month
trend_df = trend_df.drop(['date','isPartial'],axis=1)
 trend_df = trend_df.groupby(['year','month'] ).max().reset_index()
if IP_trend is None:
IP_trend = trend_df
else:
IP_trend = IP_trend.merge(trend_df,on=['year','month'],how='left')
 return IP_trend<categorify>
avg_tta_score('bd-efficientnet-b03-2015val-psu3-sz-300' )
%%time
lag_months = [1,2,3,6,12]
lag_features_list = ['item_cnt_month']
on_key = ['shop_id','item_id','date_block_num']
fillna = [0]
dtypes = ['int8']
dataset = add_lag_v3(dataset,None,lag_months,lag_features_list,on_key,fillna,dtypes )<categorify>
data = get_data(seed=0, size=IMG_SIZE, bs=BS)
 learn = cnn_learner(data, models.densenet201, model_dir=".", lin_ftrs=[2048], callback_fns=[BnFreeze], pretrained=False)
 make_or_preds(
 model_name='bd-densenet201-2015val-psu3-2019-val',
learner=learn,
model_path='.. /input/bd-densenet201-2015val-psu3-2019-val/best_model',
expected_val=0.690059677041891
)
%%time
lag_months = [1]
lag_features_list = ['avg_price']
on_key = ['shop_id','item_id','date_block_num']
fillna = [0]
dtypes = ['float32']
dataset = add_lag_v3(dataset,None,lag_months,lag_features_list,on_key,fillna,dtypes )<drop_column>
data = get_data(seed=0, size=IMG_SIZE, bs=BS, tfms=((),(flip_lr(p=1))))
learn = cnn_learner(data, models.densenet201, model_dir=".", lin_ftrs=[2048], callback_fns=[BnFreeze], pretrained=False)
 make_or_preds(
 model_name='bd-densenet201-2015val-psu3-2019-val-flip',
learner=learn,
model_path='.. /input/bd-densenet201-2015val-psu3-2019-val/best_model',
expected_val=0.690059677041891
)
dataset = add_first_last_sale(dataset,df,['shop_id','item_id'],'pair')
 dataset = dataset.drop(['pair_first_sold','pair_last_sold'],axis=1 )<categorify>
avg_tta_score('bd-densenet201-2015val-psu3-2019-val' )
%%time
item_month_sale = create_mean_encoding_table(dataset, ['item_id'], 'item')
 lag_months = [1]
lag_features_list = ['item_avg_cnt_month', 'item_avg_revenue_month', 'item_avg_price_month']
on_key = ['item_id','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,item_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['item_avg_cnt_month','item_avg_revenue_month', 'item_avg_price_month'],axis=1)
 del item_month_sale
gc.collect()<drop_column>
columns = ['x_0', 'x_1', 'x_2', 'x_3', 'x_4']
eff_b3_300_val_preds_no_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-300_val_preds.csv' ).sort_values('id_code')
 eff_b3_300_val_preds_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-300-flip_val_preds.csv' ).sort_values('id_code')
 eff_b3_300_val_preds =(eff_b3_300_val_preds_flip[columns] + eff_b3_300_val_preds_no_flip[columns])/ 2
eff_b3_256_val_preds_no_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-256_val_preds.csv' ).sort_values('id_code')
 eff_b3_256_val_preds_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-256_val_preds.csv' ).sort_values('id_code')
 eff_b3_256_val_preds =(eff_b3_256_val_preds_flip[columns] + eff_b3_256_val_preds_no_flip[columns])/ 2
densenet_201_val_preds_no_flip = pd.read_csv('bd-densenet201-2015val-psu3-2019-val_val_preds.csv' ).sort_values('id_code')
 densenet_201_val_preds_flip = pd.read_csv('bd-densenet201-2015val-psu3-2019-val-flip_val_preds.csv' ).sort_values('id_code')
 densenet_201_val_preds =(densenet_201_val_preds_flip[columns] + densenet_201_val_preds_no_flip[columns])/ 2
%%time
dataset = add_first_last_sale(dataset,df,['item_id'],'item')
 dataset = dataset.drop(['item_first_sold','item_last_sold'],axis=1 )<data_type_conversions>
val_preds_avg =(
 eff_b3_300_val_preds.values * 0.4 +
eff_b3_256_val_preds.values * 0.4 +
densenet_201_val_preds.values * 0.2
)
dataset = pd.merge(dataset,items[['item_id','item_category_id']],how='left',on=['item_id'])
 dataset['item_category_id'] = dataset['item_category_id'].astype('int16' )<save_to_csv>
preds = []
thres = []
for i in np.arange(0, 1, 0.005):
thres.append(i)
 preds.append(cohen_kappa_score(
 eff_b3_300_val_preds_flip.sort_values('id_code' ).diagnosis,
get_output_preds(( val_preds_avg > i)) , weights='quadratic'))
try:
item_categories_plus = pd.read_csv(tables_path + 'item_categories_plus.csv')
 except FileNotFoundError:
item_categories_plus = item_categories.copy()
trans = Translator()
item_categories_plus['item_category_name_en'] = item_categories_plus['item_category_name'].apply(lambda name: trans.translate(name,dest='en' ).text)
 item_categories_plus
item_categories_plus.to_csv(tables_path + 'item_categories_plus.csv')
 item_categories_plus['main_category'] = item_categories_plus['item_category_name_en'].apply(lambda name: name.split('-')[0].strip())
 item_categories_plus['sub_category'] = item_categories_plus['item_category_name_en'].apply(
 lambda name:(name.split('-')[1].strip())if len(name.split('-')) >1 else(name.split('-')[0].strip()))
item_categories_plus.loc[24,'main_category'] = 'Games'
item_categories_plus.loc[71,'main_category'] = 'Platic Bags and Gift'
item_categories_plus.loc[78,'main_category'] = 'Program'
item_categories_plus.loc[32,'main_category'] = 'Payment cards'
item_categories_plus.loc[32,'sub_category'] = '(Movies, Music, Games)'
item_categories_plus.loc[36,'main_category'] = 'Payment cards'
item_categories_plus.loc[37:41,'main_category'] = 'Movies'
main_category_encoder = LabelEncoder()
item_categories_plus['main_category_enc'] = main_category_encoder.fit_transform(item_categories_plus['main_category'] ).astype('int8')
 sub_category_encoder = LabelEncoder()
item_categories_plus['sub_category_enc'] = sub_category_encoder.fit_transform(item_categories_plus['sub_category'] ).astype('int8')
 dataset = pd.merge(dataset,item_categories_plus[['item_category_id','main_category_enc','sub_category_enc']],how='left',on=['item_category_id'])
 del item_categories_plus
gc.collect()<categorify>
val_preds_and_threshold = sorted(list(zip(preds, thres)) , key=lambda x: x[0], reverse=True )
<categorify>
best_thresh = 0.45
%%time
main_cate_month_sale = create_mean_encoding_table(dataset, ['main_category_enc'], 'main_cate')
 lag_months = [1]
lag_features_list = ['main_cate_avg_cnt_month', 'main_cate_avg_revenue_month', 'main_cate_avg_price_month']
on_key = ['main_category_enc','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,main_cate_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['main_cate_avg_cnt_month','main_cate_avg_revenue_month', 'main_cate_avg_price_month'],axis=1)
 del main_cate_month_sale
gc.collect()<categorify>
columns = ['x_0', 'x_1', 'x_2', 'x_3', 'x_4']
eff_b3_300_test_preds_no_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-300_test_preds.csv')
 eff_b3_300_test_preds_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-300-flip_test_preds.csv')
 eff_b3_300_test_preds =(eff_b3_300_test_preds_flip[columns] + eff_b3_300_test_preds_no_flip[columns])/ 2
eff_b3_256_test_preds_no_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-256_test_preds.csv')
 eff_b3_256_test_preds_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-256-flip_test_preds.csv')
 eff_b3_256_test_preds =(eff_b3_256_test_preds_flip[columns] + eff_b3_256_test_preds_no_flip[columns])/ 2
densenet_201_test_preds_no_flip = pd.read_csv('bd-densenet201-2015val-psu3-2019-val_test_preds.csv' ).sort_values('id_code')
 densenet_201_test_preds_flip = pd.read_csv('bd-densenet201-2015val-psu3-2019-val-flip_test_preds.csv' ).sort_values('id_code')
 densenet_201_test_preds =(densenet_201_test_preds_flip[columns] + densenet_201_test_preds_no_flip[columns])/ 2
%%time
sub_cate_month_sale = create_mean_encoding_table(dataset, ['sub_category_enc'], 'sub_cate')
 lag_months = [1]
lag_features_list = ['sub_cate_avg_cnt_month', 'sub_cate_avg_revenue_month', 'sub_cate_avg_price_month']
on_key = ['sub_category_enc','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,sub_cate_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['sub_cate_avg_cnt_month','sub_cate_avg_revenue_month', 'sub_cate_avg_price_month'],axis=1)
 del sub_cate_month_sale
gc.collect()<create_dataframe>
test_preds_avg =(
 eff_b3_300_test_preds.values * 0.4 +
eff_b3_256_test_preds.values * 0.4 +
densenet_201_test_preds.values * 0.2
)
%%time
dataset = first_sale_cnt(dataset,dataset,['item_category_id'],'cate')
 dataset = first_sale_cnt(dataset,dataset,['main_category_enc'],'main_cate')
 dataset = first_sale_cnt(dataset,dataset,['sub_category_enc'],'sub_cate' )<feature_engineering>
sample_df.diagnosis = get_output_preds(( test_preds_avg > best_thresh))
date_features = df.drop_duplicates(['date_block_num'])[['date','date_block_num']]
date_features['year'] = date_features['date'].apply(lambda d: d.split('.')[2])
 date_features['month'] = date_features['date'].apply(lambda d: d.split('.')[1])
 date_features = date_features.drop('date',axis=1 ).reset_index(drop=True)
 test_date = [34,2015,11]
date_features.loc[34] = test_date
dataset = pd.merge(dataset,date_features,how='left',on=['date_block_num'])
 dataset[['month','year']] = dataset[['month','year']].astype('int16')
<categorify>
sample_df.to_csv('submission.csv', index=False )
<save_to_csv><EOS> 
pd.read_csv('submission.csv' ).diagnosis.value_counts()