Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<load_pretrained>
%matplotlib inline
warnings.filterwarnings("ignore" )
predictions = []
encoder_name = ["resnet34"]
model_name = ["uresnet34v3"]
model_dir = ["resnet34"]
models = []
for i in range(len(model_name)) :
ckpt_path = ".. /input/{}/{}.pth".format(model_dir[i],model_name[i])
 device = torch.device("cuda")
 model = Unet(encoder_name=encoder_name[i],encoder_weights = None,classes=4,activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path,map_location=lambda storage,loc:storage)
 model.load_state_dict(state["state_dict"])
 models.append(model )<predict_on_test>
def encode_data(df):
df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'])
 def feature_importance(forest, X_train):
ranked_list = []
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
print("Feature ranking:")
 for f in range(X_train.shape[1]):
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 return ranked_list
predictions = []
for i,(batch,batch_hTTA,batch_vTTA)in enumerate(tqdm(zip(testset,testset_hTTA,testset_vTTA))):
fnames, images = batch
fnames, images_hTTA = batch_hTTA
fnaems, images_vTTA = batch_vTTA
batch_preds = 0
for model in models:
pred = torch.sigmoid(model(images.to(device)) ).detach().cpu().numpy()
pred_hTTA = torch.sigmoid(model(images_hTTA.to(device)) ).detach().cpu().numpy()
pred_vTTA = torch.sigmoid(model(images_vTTA.to(device)) ).detach().cpu().numpy()
batch_preds +=(pred+pred_hTTA[:,:,:,::-1]+pred_vTTA[:,:,::-1,:])/3
batch_preds/= len(models)
 for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv("submission.csv", index=False )<set_options>
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop(['Id'], axis=1, inplace=True)
 return df
%reload_ext autoreload
%autoreload 2
%matplotlib inline
warnings.filterwarnings("ignore")
 fastai.__version__<define_variables>
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df
nfolds = 1
bs = 4
n_cls = 4
noise_th = 2000
TEST = '.. /input/severstal-steel-defect-detection/test_images/'
BASE = '.. /input/severstal-fast-ai-256x256-crops/'
torch.backends.cudnn.benchmark = True<import_modules>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ids = test.Id
class Hcolumns(nn.Module):
def __init__(self, hooks:Collection[Hook], nc:Collection[int]=None):
super(Hcolumns,self ).__init__()
self.hooks = hooks
self.n = len(self.hooks)
 self.factorization = None
if nc is not None:
self.factorization = nn.ModuleList()
for i in range(self.n):
self.factorization.append(nn.Sequential(
 conv2d(nc[i],nc[-1],3,padding=1,bias=True),
conv2d(nc[-1],nc[-1],3,padding=1,bias=True)))
 def forward(self, x:Tensor):
n = len(self.hooks)
 out = [F.interpolate(self.hooks[i].stored if self.factorization is None
else self.factorization[i](self.hooks[i].stored), scale_factor=2**(self.n-i),
mode='bilinear',align_corners=False)for i in range(self.n)] + [x]
return torch.cat(out, dim=1)
 class DynamicUnet_Hcolumns(SequentialEx):
"Create a U-Net from a given architecture."
def __init__(self, encoder:nn.Module, n_classes:int, blur:bool=False, blur_final=True,
self_attention:bool=False,
y_range:Optional[Tuple[float,float]]=None,
last_cross:bool=True, bottle:bool=False, **kwargs):
imsize =(256,256)
 sfs_szs = model_sizes(encoder, size=imsize)
 sfs_idxs = list(reversed(_get_sfs_idxs(sfs_szs)))
 self.sfs = hook_outputs([encoder[i] for i in sfs_idxs])
 x = dummy_eval(encoder, imsize ).detach()
ni = sfs_szs[-1][1]
middle_conv = nn.Sequential(conv_layer(ni, ni*2, **kwargs),
conv_layer(ni*2, ni, **kwargs)).eval()
x = middle_conv(x)
 layers = [encoder, batchnorm_2d(ni), nn.ReLU() , middle_conv]
self.hc_hooks = [Hook(layers[-1], _hook_inner, detach=False)]
hc_c = [x.shape[1]]
for i,idx in enumerate(sfs_idxs):
not_final = i!=len(sfs_idxs)-1
up_in_c, x_in_c = int(x.shape[1]), int(sfs_szs[idx][1])
 do_blur = blur and(not_final or blur_final)
 sa = self_attention and(i==len(sfs_idxs)-3)
 unet_block = UnetBlock(up_in_c, x_in_c, self.sfs[i], final_div=not_final,
blur=blur, self_attention=sa, **kwargs ).eval()
layers.append(unet_block)
 x = unet_block(x)
 self.hc_hooks.append(Hook(layers[-1], _hook_inner, detach=False))
hc_c.append(x.shape[1])
 ni = x.shape[1]
if imsize != sfs_szs[0][-2:]: layers.append(PixelShuffle_ICNR(ni, **kwargs))
if last_cross:
layers.append(MergeLayer(dense=True))
ni += in_channels(encoder)
 layers.append(res_block(ni, bottle=bottle, **kwargs))
hc_c.append(ni)
 layers.append(Hcolumns(self.hc_hooks, hc_c))
layers += [conv_layer(ni*len(hc_c), n_classes, ks=1, use_activ=False, **kwargs)]
if y_range is not None: layers.append(SigmoidRange(*y_range))
super().__init__(*layers)
 def __del__(self):
if hasattr(self, "sfs"): self.sfs.remove()
def unet_learner(data:DataBunch, arch:Callable, pretrained:bool=True, blur_final:bool=True,
norm_type:Optional[NormType]=NormType, split_on:Optional[SplitFuncOrIdxList]=None,
blur:bool=False, self_attention:bool=False, y_range:Optional[Tuple[float,float]]=None,
last_cross:bool=True, bottle:bool=False, cut:Union[int,Callable]=None,
hypercolumns=True, **learn_kwargs:Any)->Learner:
"Build Unet learner from `data` and `arch`."
meta = cnn_config(arch)
 body = create_body(arch, pretrained, cut)
 M = DynamicUnet_Hcolumns if hypercolumns else DynamicUnet
model = to_device(M(body, n_classes=data.c, blur=blur, blur_final=blur_final,
self_attention=self_attention, y_range=y_range, norm_type=norm_type,
last_cross=last_cross, bottle=bottle), data.device)
 learn = Learner(data, model, **learn_kwargs)
 learn.split(ifnone(split_on, meta['split']))
if pretrained: learn.freeze()
apply_init(model[2], nn.init.kaiming_normal_)
 return learn
class SegmentationLabelList(SegmentationLabelList):
def open(self, fn): return open_mask(fn, div=True)
 class SegmentationItemList(SegmentationItemList):
_label_cls = SegmentationLabelList
def transform(self, tfms:Optional[Tuple[TfmList,TfmList]]=(None,None), **kwargs):
if not tfms: tfms=(None,None)
 assert is_listy(tfms)and len(tfms)== 2
self.train.transform(tfms[0], **kwargs)
 self.valid.transform(tfms[1], **kwargs)
 kwargs['tfm_y'] = False
if self.test: self.test.transform(tfms[1], **kwargs)
 return self
fastai.data_block.ItemLists.transform = transform
def open_mask(fn:PathOrStr, div:bool=True, convert_mode:str='L', cls:type=ImageSegment,
after_open:Callable=None)->ImageSegment:
with warnings.catch_warnings() :
warnings.simplefilter("ignore", UserWarning)
 x = PIL.Image.open(fn ).convert(convert_mode)
 if after_open: x = after_open(x)
 x = pil2tensor(x,np.float32)
 return cls(x )<categorify>
def process_df(df_):
encode_data(df_)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )
def enc2mask(encs, shape=(1600,256)) :
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for m,enc in enumerate(encs):
if isinstance(enc,np.float)and np.isnan(enc): continue
s = enc.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1 + m
return img.reshape(shape ).T
def mask2enc(mask, n=n_cls):
pixels = mask.T.flatten()
encs = []
for i in range(1,n+1):
p =(pixels == i ).astype(np.int8)
 if p.sum() == 0: encs.append('')
 else:
p = np.concatenate([[0], p, [0]])
 runs = np.where(p[1:] != p[:-1])[0] + 1
runs[1::2] -= runs[::2]
encs.append(' '.join(str(x)for x in runs))
return encs<load_pretrained>
train['dependency'] = np.sqrt(train['SQBdependency'])
 test['dependency'] = np.sqrt(test['SQBdependency'])
 train.loc[train['edjefa'] == "no", "edjefa"] = 0
train.loc[train['edjefe'] == "no", "edjefe"] = 0
test.loc[test['edjefa'] == "no", "edjefa"] = 0
test.loc[test['edjefe'] == "no", "edjefe"] = 0
train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "edjefa"] = train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "escolari"]
train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "edjefe"] = train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "edjefa"] = test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "edjefe"] = test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "escolari"]
train.loc[train['edjefa'] == "yes", "edjefa"] = 4
train.loc[train['edjefe'] == "yes", "edjefe"] = 4
test.loc[test['edjefa'] == "yes", "edjefa"] = 4
test.loc[test['edjefe'] == "yes", "edjefe"] = 4
train['edjefe'] = train['edjefe'].astype("int")
 train['edjefa'] = train['edjefa'].astype("int")
 test['edjefe'] = test['edjefe'].astype("int")
 test['edjefa'] = test['edjefa'].astype("int")
 train['edjef'] = np.max(train[['edjefa','edjefe']], axis=1)
 test['edjef'] = np.max(test[['edjefa','edjefe']], axis=1)
 train['v2a1']=train['v2a1'].fillna(0)
 test['v2a1']=test['v2a1'].fillna(0)
 test['v18q1']=test['v18q1'].fillna(0)
 train['v18q1']=train['v18q1'].fillna(0)
 train['rez_esc']=train['rez_esc'].fillna(0)
 test['rez_esc']=test['rez_esc'].fillna(0)
 train.loc[train.meaneduc.isnull() , "meaneduc"] = 0
train.loc[train.SQBmeaned.isnull() , "SQBmeaned"] = 0
test.loc[test.meaneduc.isnull() , "meaneduc"] = 0
test.loc[test.SQBmeaned.isnull() , "SQBmeaned"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "v14a"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "sanitario1"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "v14a"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "sanitario1"] = 0
rles,ids_test = [],[]
learns = []
for fold in range(nfolds):
learn = unet_learner(data, models.resnet34, pretrained=False)
 learn.model.load_state_dict(torch.load(Path(BASE)/f'models/fold{fold}.pth')['model'])
 learns.append(learn)
 with zipfile.ZipFile('pred.zip', 'w')as archive_out:
def to_mask(yp, y, id):
name = id + '.png'
save_img(yp[:,:,1:],name,archive_out)
 yp = np.argmax(yp, axis=-1)
 for i in range(n_cls):
idxs = yp == i+1
if idxs.sum() < noise_th: yp[idxs] = 0
encs = mask2enc(yp)
 for i, enc in enumerate(encs):
ids_test.append(id + '.jpg_' + str(i+1))
rles.append(enc)
 model_pred(learns,to_mask,DatasetType.Test)
 sub_df = pd.DataFrame({'ImageId_ClassId': ids_test, 'EncodedPixels': rles})
 sub_df.sort_values(by='ImageId_ClassId' ).to_csv('submission.csv', index=False )<set_options>
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_
%reload_ext autoreload
%autoreload 2
%matplotlib inline
warnings.filterwarnings("ignore")
 fastai.__version__<define_variables>
train, test = train_test_apply_func(train, test, convert_OHE2LE )
nfolds = 1
bs = 4
n_cls = 4
noise_th = 2000
TEST = '.. /input/severstal-steel-defect-detection/test_images/'
BASE = '.. /input/severstal-fast-ai-256x256-crops/'
torch.backends.cudnn.benchmark = True<import_modules>
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )
class Hcolumns(nn.Module):
def __init__(self, hooks:Collection[Hook], nc:Collection[int]=None):
super(Hcolumns,self ).__init__()
self.hooks = hooks
self.n = len(self.hooks)
 self.factorization = None
if nc is not None:
self.factorization = nn.ModuleList()
for i in range(self.n):
self.factorization.append(nn.Sequential(
 conv2d(nc[i],nc[-1],3,padding=1,bias=True),
conv2d(nc[-1],nc[-1],3,padding=1,bias=True)))
 def forward(self, x:Tensor):
n = len(self.hooks)
 out = [F.interpolate(self.hooks[i].stored if self.factorization is None
else self.factorization[i](self.hooks[i].stored), scale_factor=2**(self.n-i),
mode='bilinear',align_corners=False)for i in range(self.n)] + [x]
return torch.cat(out, dim=1)
 class DynamicUnet_Hcolumns(SequentialEx):
"Create a U-Net from a given architecture."
def __init__(self, encoder:nn.Module, n_classes:int, blur:bool=False, blur_final=True,
self_attention:bool=False,
y_range:Optional[Tuple[float,float]]=None,
last_cross:bool=True, bottle:bool=False, **kwargs):
imsize =(256,256)
 sfs_szs = model_sizes(encoder, size=imsize)
 sfs_idxs = list(reversed(_get_sfs_idxs(sfs_szs)))
 self.sfs = hook_outputs([encoder[i] for i in sfs_idxs])
 x = dummy_eval(encoder, imsize ).detach()
ni = sfs_szs[-1][1]
middle_conv = nn.Sequential(conv_layer(ni, ni*2, **kwargs),
conv_layer(ni*2, ni, **kwargs)).eval()
x = middle_conv(x)
 layers = [encoder, batchnorm_2d(ni), nn.ReLU() , middle_conv]
self.hc_hooks = [Hook(layers[-1], _hook_inner, detach=False)]
hc_c = [x.shape[1]]
for i,idx in enumerate(sfs_idxs):
not_final = i!=len(sfs_idxs)-1
up_in_c, x_in_c = int(x.shape[1]), int(sfs_szs[idx][1])
 do_blur = blur and(not_final or blur_final)
 sa = self_attention and(i==len(sfs_idxs)-3)
 unet_block = UnetBlock(up_in_c, x_in_c, self.sfs[i], final_div=not_final,
blur=blur, self_attention=sa, **kwargs ).eval()
layers.append(unet_block)
 x = unet_block(x)
 self.hc_hooks.append(Hook(layers[-1], _hook_inner, detach=False))
hc_c.append(x.shape[1])
 ni = x.shape[1]
if imsize != sfs_szs[0][-2:]: layers.append(PixelShuffle_ICNR(ni, **kwargs))
if last_cross:
layers.append(MergeLayer(dense=True))
ni += in_channels(encoder)
 layers.append(res_block(ni, bottle=bottle, **kwargs))
hc_c.append(ni)
 layers.append(Hcolumns(self.hc_hooks, hc_c))
layers += [conv_layer(ni*len(hc_c), n_classes, ks=1, use_activ=False, **kwargs)]
if y_range is not None: layers.append(SigmoidRange(*y_range))
super().__init__(*layers)
 def __del__(self):
if hasattr(self, "sfs"): self.sfs.remove()
def unet_learner(data:DataBunch, arch:Callable, pretrained:bool=True, blur_final:bool=True,
norm_type:Optional[NormType]=NormType, split_on:Optional[SplitFuncOrIdxList]=None,
blur:bool=False, self_attention:bool=False, y_range:Optional[Tuple[float,float]]=None,
last_cross:bool=True, bottle:bool=False, cut:Union[int,Callable]=None,
hypercolumns=True, **learn_kwargs:Any)->Learner:
"Build Unet learner from `data` and `arch`."
meta = cnn_config(arch)
 body = create_body(arch, pretrained, cut)
 M = DynamicUnet_Hcolumns if hypercolumns else DynamicUnet
model = to_device(M(body, n_classes=data.c, blur=blur, blur_final=blur_final,
self_attention=self_attention, y_range=y_range, norm_type=norm_type,
last_cross=last_cross, bottle=bottle), data.device)
 learn = Learner(data, model, **learn_kwargs)
 learn.split(ifnone(split_on, meta['split']))
if pretrained: learn.freeze()
apply_init(model[2], nn.init.kaiming_normal_)
 return learn
class SegmentationLabelList(SegmentationLabelList):
def open(self, fn): return open_mask(fn, div=True)
 class SegmentationItemList(SegmentationItemList):
_label_cls = SegmentationLabelList
def transform(self, tfms:Optional[Tuple[TfmList,TfmList]]=(None,None), **kwargs):
if not tfms: tfms=(None,None)
 assert is_listy(tfms)and len(tfms)== 2
self.train.transform(tfms[0], **kwargs)
 self.valid.transform(tfms[1], **kwargs)
 kwargs['tfm_y'] = False
if self.test: self.test.transform(tfms[1], **kwargs)
 return self
fastai.data_block.ItemLists.transform = transform
def open_mask(fn:PathOrStr, div:bool=True, convert_mode:str='L', cls:type=ImageSegment,
after_open:Callable=None)->ImageSegment:
with warnings.catch_warnings() :
warnings.simplefilter("ignore", UserWarning)
 x = PIL.Image.open(fn ).convert(convert_mode)
 if after_open: x = after_open(x)
 x = pil2tensor(x,np.float32)
 return cls(x )<categorify>
train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0)
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
extract_features(train)
 extract_features(test )
def enc2mask(encs, shape=(1600,256)) :
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for m,enc in enumerate(encs):
if isinstance(enc,np.float)and np.isnan(enc): continue
s = enc.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1 + m
return img.reshape(shape ).T
def mask2enc(mask, n=n_cls):
pixels = mask.T.flatten()
encs = []
for i in range(1,n+1):
p =(pixels == i ).astype(np.int8)
 if p.sum() == 0: encs.append('')
 else:
p = np.concatenate([[0], p, [0]])
 runs = np.where(p[1:] != p[:-1])[0] + 1
runs[1::2] -= runs[::2]
encs.append(' '.join(str(x)for x in runs))
return encs<load_pretrained>
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'female', ]
instlevel_cols = [s for s in train.columns.tolist() if 'instlevel' in s]
needless_cols.extend(instlevel_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )
rles,ids_test = [],[]
learns = []
for fold in range(nfolds):
learn = unet_learner(data, models.resnet34, pretrained=False)
 learn.model.load_state_dict(torch.load(Path(BASE)/f'models/fold{fold}.pth')['model'])
 learns.append(learn)
 with zipfile.ZipFile('pred.zip', 'w')as archive_out:
def to_mask(yp, y, id):
name = id + '.png'
save_img(yp[:,:,1:],name,archive_out)
 yp = np.argmax(yp, axis=-1)
 for i in range(n_cls):
idxs = yp == i+1
if idxs.sum() < noise_th: yp[idxs] = 0
encs = mask2enc(yp)
 for i, enc in enumerate(encs):
ids_test.append(id + '.jpg_' + str(i+1))
rles.append(enc)
 model_pred(learns,to_mask,DatasetType.Test)
 sub_df = pd.DataFrame({'ImageId_ClassId': ids_test, 'EncodedPixels': rles})
 sub_df.sort_values(by='ImageId_ClassId' ).to_csv('submission.csv', index=False )<set_options>
def split_data(train, y, households, test_percentage=0.20, seed=None):
train2 = train.copy()
cv_hhs = np.random.choice(households, size=int(len(households)* test_percentage), replace=False)
 cv_idx = np.isin(households, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
return X_train, y_train, X_test, y_test
%matplotlib inline<load_from_csv>
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=1977)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.15), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
X_train = train2
y_train = y
train_households = X_train.idhogar
train_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<feature_engineering>
extra_drop_features = [
'geo_manual_elec_LE_2',
'agg18_estadocivil5_COUNT',
'agg18_estadocivil7_COUNT',
'geo_pared_LE_5',
'agg18_estadocivil6_COUNT',
'geo_pared_LE_4',
'agg18_parentesco10_COUNT',
'num_over_18',
'agg18_estadocivil1_MEAN',
'agg18_estadocivil4_COUNT',
'r4t3_to_tamhog',
'agg18_estadocivil3_COUNT',
'geo_pared_LE_3',
'geo_pared_LE_6',
'geo_pared_LE_0',
'geo_pared_LE_7',
'agg18_parentesco7_COUNT',
'agg18_parentesco10_MEAN',
'agg18_parentesco8_MEAN',
'rez_esc',
'hacdor',
'hacapo',
'fe_people_weird_stat',
'geo_hogar_mayor',
'geo_eviv_LE_0',
'geo_epared_LE_0',
'geo_epared_LE_2',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_0',
'agg18_parentesco9_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco11_COUNT',
'parentesco_LE',
'agg18_parentesco6_MEAN',
'agg18_parentesco6_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco4_MEAN',
'agg18_parentesco4_COUNT',
'geo_energcocinar_LE_1',
'agg18_parentesco3_COUNT',
'geo_energcocinar_LE_2',
'agg18_parentesco2_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco12_MEAN',
'agg18_parentesco12_COUNT',
] + ['geo_hogar_nin', 'geo_manual_elec_LE_3']
train_data['train'] = 1
test_data['train'] = 0<concatenate>
lgb_drop_cols = extra_drop_features + ["idhogar", 'parentesco1']
try:
X_train.drop(lgb_drop_cols, axis=1, inplace=True)
 X_test.drop(lgb_drop_cols, axis=1, inplace=True)
 except:
print("Error dropping columns" )
full_data = pd.concat([train_data, test_data], sort = False, ignore_index = True )<categorify>
opt_parameters = {'colsample_bytree': 0.88, 'min_child_samples': 90, 'num_leaves': 16, 'subsample': 0.94, 'reg_lambda': 0.5, }
opt_parameters = {'colsample_bytree': 0.88, 'min_child_samples': 95, 'num_leaves': 25, 'subsample': 0.94, 'reg_lambda': 0.5, }
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":800,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]
full_data['SalePrice'].replace(np.nan, 1, inplace = True )<sort_values>
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):
X_train, y_train, X_test, y_test = split_data(X, y, households=train_households, seed=1972)
 fit_params["eval_set"] = [(X_train,y_train),(X_test,y_test)]
if sample_weight is not None:
estimator.fit(X_train, y_train, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X_train, y_train, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self
total = full_data.isnull().sum().sort_values(ascending = False)
 percent =(full_data.isnull().sum() /full_data.isnull().count() ).sort_values(ascending = False)
 missing_data = pd.concat([total, percent], axis = 1, keys = ['Total', 'Percent'])
 missing_data.head(20)
<categorify>
clfs = []
for i in range(30):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_train, y_train, **fit_params)
 clf_final = vc.estimators_[0]
full_data.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence'], axis = 1, inplace = True)
 full_data['FireplaceQu'].fillna('No', inplace = True)
 full_data["LotFrontage"] = full_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageFinish', 'GarageQual', 'GarageCond', 'GarageType'):
full_data[col].fillna('No', inplace = True)
 full_data['GarageYrBlt'].fillna(0, inplace = True)
 for col in('BsmtCond', 'BsmtExposure', 'BsmtQual', 'BsmtFinType1', 'BsmtFinType2'):
full_data[col].fillna('No', inplace = True)
 full_data['MasVnrType'].fillna('No', inplace = True)
 full_data['MasVnrArea'].fillna(0, inplace = True)
 full_data['MSZoning'].fillna('RL', inplace = True)
 full_data['BsmtFullBath'].fillna(0, inplace = True)
 full_data['BsmtHalfBath'].fillna(0, inplace = True)
 full_data.drop(['Utilities'], axis = 1, inplace = True)
 full_data['Functional'].fillna('Typ', inplace = True)
 full_data['GarageArea'].fillna(0, inplace = True)
 full_data['TotalBsmtSF'].fillna(0, inplace = True)
 full_data['Electrical'].fillna('SBrkr', inplace = True)
 full_data['Electrical'].replace(['Mix', 'FuseP','FuseF'], ['SBrkr', 'SBrkr', 'SBrkr'], inplace = True)
 full_data.drop(['Exterior1st', 'Exterior2nd'], axis = 1, inplace = True)
 full_data['KitchenQual'].fillna('TA', inplace = True)
 full_data['GarageCars'].fillna(0, inplace = True)
 full_data.drop(['BsmtUnfSF'], axis = 1, inplace = True)
 full_data.drop(['BsmtFinSF1', 'BsmtFinSF2'], axis = 1, inplace = True)
 full_data.drop(['Neighborhood', 'Street', 'Condition2', 'RoofMatl', 'Heating'], axis = 1, inplace = True)
 full_data['SaleType'].fillna('WD', inplace = True )<feature_engineering>
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))
full_data['GrLivArea'] = np.log(full_data['GrLivArea'])
 full_data['LotArea'] = np.log1p(full_data['LotArea'])
 full_data['LowQualFinSF'] = np.log1p(full_data['LowQualFinSF'])
 full_data['BsmtHalfBath'] = np.log1p(full_data['BsmtHalfBath'])
 full_data['KitchenAbvGr'] = np.log1p(full_data['KitchenAbvGr'])
 full_data['GarageYrBlt'] = np.log1p(full_data['GarageYrBlt'])
 full_data['EnclosedPorch'] = np.log1p(full_data['EnclosedPorch'])
 full_data['3SsnPorch'] = np.log1p(full_data['3SsnPorch'])
 full_data['ScreenPorch'] = np.log1p(full_data['ScreenPorch'])
 full_data['PoolArea'] = np.log1p(full_data['PoolArea'])
 full_data['MiscVal'] = np.log1p(full_data['MiscVal'] )<define_variables>
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))
cat_to_num = {
'HouseStyle' : {'1Story' : 1, '1.5Unf' : 2, '1.5Fin' : 3, '2Story' : 4, '2.5Unf' : 5,
'2.5Fin' : 6, 'SFoyer' : 7, 'SLvl' : 8},
'ExterQual' : {'Ex' : 10, 'Gd' : 8, 'TA' : 6, 'Fa' : 4, 'Po' : 2},
'ExterCond' : {'Ex' : 10, 'Gd' : 8, 'TA' : 6, 'Fa' : 4, 'Po' : 2},
'BsmtQual' : {'Ex' : 10, 'Gd' : 8, 'TA' : 6, 'Fa' : 4, 'Po' : 2, 'No' : 0},
'BsmtCond' : {'Ex' : 10, 'Gd' : 8, 'TA' : 6, 'Fa' : 4, 'Po' : 2, 'No' : 0},
'BsmtExposure' : {'Gd' : 10, 'Av' : 7, 'Mn' : 4, 'No' : 0},
'BsmtFinType1' : {'GLQ' : 10, 'ALQ' : 8, 'BLQ' : 6, 'Rec' : 5, 'LwQ' : 4, 'Unf' : 2, 'No' : 0},
'BsmtFinType2' : {'GLQ' : 10, 'ALQ' : 8, 'BLQ' : 6, 'Rec' : 5, 'LwQ' : 4, 'Unf' : 2, 'No' : 0},
'HeatingQC' : {'Ex' : 10, 'Gd' : 8, 'TA' : 6, 'Fa' : 4, 'Po' : 2},
'KitchenQual' : {'Ex' : 10, 'Gd' : 8, 'TA' : 6, 'Fa' : 4, 'Po' : 2},
'FireplaceQu' : {'Ex' : 10, 'Gd' : 8, 'TA' : 6, 'Fa' : 4, 'Po' : 2, 'No' : 0},
'GarageQual' : {'Ex' : 10, 'Gd' : 8, 'TA' : 6, 'Fa' : 4, 'Po' : 2, 'No' : 0},
'GarageCond' : {'Ex' : 10, 'Gd' : 8, 'TA' : 6, 'Fa' : 4, 'Po' : 2, 'No' : 0}
} 
<define_variables>
ranked_features = feature_importance(clf_final, X_train )
full_data.replace(cat_to_num, inplace = True )<count_values>
y_subm = pd.DataFrame()
y_subm['Id'] = test_ids
threshold = 50
cat_columns = list(full_data.select_dtypes(include = ['object', 'category'] ).columns)
 for col in cat_columns:
vc = full_data[col].value_counts()
val_remove = vc[vc <= threshold].index.values
full_data[col].loc[full_data[col].isin(val_remove)] = 'Other'<categorify>
y_subm['Target'] = clf_final.predict(test[X_train.columns])+ 1
vc.voting = 'soft'
y_subm_soft = y_subm.copy(deep=True)
 y_subm_soft['Target'] = vc.predict(test[X_train.columns])+ 1
vc.voting = 'hard'
y_subm_hard = y_subm.copy(deep=True)
 y_subm_hard['Target'] = vc.predict(test[X_train.columns])+ 1
<drop_column><EOS> 
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_soft = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_hard = 'submission_hard_LGB_{:.4f}_{}.csv'.format(global_score_hard, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_0forNonHeads = 'submission_0forNonHead_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm.to_csv(sub_file, index=False)
 y_subm_soft.to_csv(sub_file_soft, index=False)
 y_subm_hard.to_csv(sub_file_hard, index=False )