Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<categorify>
sns.set(font_scale=2.2)
 plt.style.use('seaborn')
 warnings.filterwarnings('ignore')
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<define_variables>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"<define_search_space>
description = [ 
("v2a1"," Monthly rent payment"), 
("hacdor"," =1 Overcrowding by bedrooms"), 
("rooms"," number of all rooms in the house"), 
("hacapo"," =1 Overcrowding by rooms"), 
("v14a"," =1 has toilet in the household"), 
("refrig"," =1 if the household has refrigerator"), 
("v18q"," owns a tablet"), 
("v18q1"," number of tablets household owns"), 
("r4h1"," Males younger than 12 years of age"), 
("r4h2"," Males 12 years of age and older"), 
("r4h3"," Total males in the household"), 
("r4m1"," Females younger than 12 years of age"), 
("r4m2"," Females 12 years of age and older"), 
("r4m3"," Total females in the household"), 
("r4t1"," persons younger than 12 years of age"), 
("r4t2"," persons 12 years of age and older"), 
("r4t3"," Total persons in the household"), 
("tamhog"," size of the household"), 
("tamviv"," number of persons living in the household"), 
("escolari"," years of schooling"), 
("rez_esc"," Years behind in school"), 
("hhsize"," household size"), 
("paredblolad"," =1 if predominant material on the outside wall is block or brick"), 
("paredzocalo"," =1 if predominant material on the outside wall is socket(wood, zinc or absbesto"), 
("paredpreb"," =1 if predominant material on the outside wall is prefabricated or cement"), 
("pareddes"," =1 if predominant material on the outside wall is waste material"), 
("paredmad"," =1 if predominant material on the outside wall is wood"), 
("paredzinc"," =1 if predominant material on the outside wall is zink"), 
("paredfibras"," =1 if predominant material on the outside wall is natural fibers"), 
("paredother"," =1 if predominant material on the outside wall is other"), 
("pisomoscer"," =1 if predominant material on the floor is mosaic ceramic terrazo"), 
("pisocemento"," =1 if predominant material on the floor is cement"), 
("pisoother"," =1 if predominant material on the floor is other"), 
("pisonatur"," =1 if predominant material on the floor is natural material"), 
("pisonotiene"," =1 if no floor at the household"), 
("pisomadera"," =1 if predominant material on the floor is wood"), 
("techozinc"," =1 if predominant material on the roof is metal foil or zink"), 
("techoentrepiso"," =1 if predominant material on the roof is fiber cement, mezzanine "), 
("techocane"," =1 if predominant material on the roof is natural fibers"), 
("techootro"," =1 if predominant material on the roof is other"), 
("cielorazo"," =1 if the house has ceiling"), 
("abastaguadentro"," =1 if water provision inside the dwelling"), 
("abastaguafuera"," =1 if water provision outside the dwelling"), 
("abastaguano"," =1 if no water provision"), 
("public"," =1 electricity from CNFL, ICE, ESPH/JASEC"), 
("planpri"," =1 electricity from private plant"), 
("noelec"," =1 no electricity in the dwelling"), 
("coopele"," =1 electricity from cooperative"), 
("sanitario1"," =1 no toilet in the dwelling"), 
("sanitario2"," =1 toilet connected to sewer or cesspool"), 
("sanitario3"," =1 toilet connected to septic tank"), 
("sanitario5"," =1 toilet connected to black hole or letrine"), 
("sanitario6"," =1 toilet connected to other system"), 
("energcocinar1"," =1 no main source of energy used for cooking(no kitchen)"), 
("energcocinar2"," =1 main source of energy used for cooking electricity"), 
("energcocinar3"," =1 main source of energy used for cooking gas"), 
("energcocinar4"," =1 main source of energy used for cooking wood charcoal"), 
("elimbasu1"," =1 if rubbish disposal mainly by tanker truck"), 
("elimbasu2"," =1 if rubbish disposal mainly by botan hollow or buried"), 
("elimbasu3"," =1 if rubbish disposal mainly by burning"), 
("elimbasu4"," =1 if rubbish disposal mainly by throwing in an unoccupied space"), 
("elimbasu5"," =1 if rubbish disposal mainly by throwing in river, creek or sea"), 
("elimbasu6"," =1 if rubbish disposal mainly other"), 
("epared1"," =1 if walls are bad"), 
("epared2"," =1 if walls are regular"), 
("epared3"," =1 if walls are good"), 
("etecho1"," =1 if roof are bad"), 
("etecho2"," =1 if roof are regular"), 
("etecho3"," =1 if roof are good"), 
("eviv1"," =1 if floor are bad"), 
("eviv2"," =1 if floor are regular"), 
("eviv3"," =1 if floor are good"), 
("dis"," =1 if disable person"), 
("male"," =1 if male"), 
("female"," =1 if female"), 
("estadocivil1"," =1 if less than 10 years old"), 
("estadocivil2"," =1 if free or coupled uunion"), 
("estadocivil3"," =1 if married"), 
("estadocivil4"," =1 if divorced"), 
("estadocivil5"," =1 if separated"), 
("estadocivil6"," =1 if widow/er"), 
("estadocivil7"," =1 if single"), 
("parentesco1"," =1 if household head"), 
("parentesco2"," =1 if spouse/partner"), 
("parentesco3"," =1 if son/doughter"), 
("parentesco4"," =1 if stepson/doughter"), 
("parentesco5"," =1 if son/doughter in law"), 
("parentesco6"," =1 if grandson/doughter"), 
("parentesco7"," =1 if mother/father"), 
("parentesco8"," =1 if father/mother in law"), 
("parentesco9"," =1 if brother/sister"), 
("parentesco10"," =1 if brother/sister in law"), 
("parentesco11"," =1 if other family member"), 
("parentesco12"," =1 if other non family member"), 
("idhogar"," Household level identifier"), 
("hogar_nin"," Number of children 0 to 19 in household"), 
("hogar_adul"," Number of adults in household"), 
("hogar_mayor"," 
("hogar_total"," 
("dependency"," Dependency rate"), 
("edjefe"," years of education of male head of household"), 
("edjefa"," years of education of female head of household"), 
("meaneduc","average years of education for adults(18+)"), 
("instlevel1"," =1 no level of education"), 
("instlevel2"," =1 incomplete primary"), 
("instlevel3"," =1 complete primary"), 
("instlevel4"," =1 incomplete academic secondary level"), 
("instlevel5"," =1 complete academic secondary level"), 
("instlevel6"," =1 incomplete technical secondary level"), 
("instlevel7"," =1 complete technical secondary level"), 
("instlevel8"," =1 undergraduate and higher education"), 
("instlevel9"," =1 postgraduate higher education"), 
("bedrooms"," number of bedrooms"), 
("overcrowding"," 
("tipovivi1"," =1 own and fully paid house"), 
("tipovivi2"," =1 own, paying in installments"), 
("tipovivi3"," =1 rented"), 
("tipovivi4"," =1 precarious"), 
("tipovivi5"," =1 other(assigned"), 
("computer"," =1 if the household has notebook or desktop computer, borrowed)"), 
("television"," =1 if the household has TV"), 
("mobilephone"," =1 if mobile phone"), 
("qmobilephone"," 
("lugar1"," =1 region Central"), 
("lugar2"," =1 region Chorotega"), 
("lugar3"," =1 region PacÃƒÂ­fico central"), 
("lugar4"," =1 region Brunca"), 
("lugar5"," =1 region Huetar AtlÃƒÂ¡ntica"), 
("lugar6"," =1 region Huetar Norte"), 
("area1"," =1 zona urbana"), 
("area2"," =2 zona rural"), 
("age"," Age in years"), 
("SQBescolari"," escolari squared"), 
("SQBage"," age squared"), 
("SQBhogar_total"," hogar_total squared"), 
("SQBedjefe"," edjefe squared"), 
("SQBhogar_nin"," hogar_nin squared"), 
("SQBovercrowding"," overcrowding squared"), 
("SQBdependency"," dependency squared"), 
("SQBmeaned"," meaned squared"), 
("agesq"," Age squared"),]
description = pd.DataFrame(description, columns=['varname', 'description'] )
batch_size = 1
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225 )<load_from_csv>
total = df_train.isnull().sum().sort_values(ascending=False)
 percent = 100 *(df_train.isnull().sum() / df_train.isnull().count() ).sort_values(ascending=False)
 missing_df = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_df.head(20 )
df = pd.read_csv(sample_submission_path )<create_dataframe>
df_train.loc[(df_train['edjefa'] == "yes")&(df_train['parentesco1'] == 1), "edjefa"] = df_train.loc[(df_train['edjefa'] == "yes")&(df_train['parentesco1'] == 1), "escolari"]
df_train.loc[(df_train['edjefe'] == "yes")&(df_train['parentesco1'] == 1), "edjefe"] = df_train.loc[(df_train['edjefe'] == "yes")&(df_train['parentesco1'] == 1), "escolari"]
df_test.loc[(df_test['edjefa'] == "yes")&(df_test['parentesco1'] == 1), "edjefa"] = df_test.loc[(df_test['edjefa'] == "yes")&(df_test['parentesco1'] == 1), "escolari"]
df_test.loc[(df_test['edjefe'] == "yes")&(df_test['parentesco1'] == 1), "edjefe"] = df_test.loc[(df_test['edjefe'] == "yes")&(df_test['parentesco1'] == 1), "escolari"]
df_train.loc[df_train['edjefa'] == "yes", "edjefa"] = 4
df_train.loc[df_train['edjefe'] == "yes", "edjefe"] = 4
df_test.loc[df_test['edjefa'] == "yes", "edjefa"] = 4
df_test.loc[df_test['edjefe'] == "yes", "edjefe"] = 4
df_train['edjef'] = np.max(df_train[['edjefa','edjefe']], axis=1)
 df_test['edjef'] = np.max(df_test[['edjefa','edjefe']], axis=1)
 df_train.loc[(df_train.v14a == 1)&(df_train.sanitario1 == 1)&(df_train.abastaguano == 0), "v14a"] = 0
df_train.loc[(df_train.v14a == 1)&(df_train.sanitario1 == 1)&(df_train.abastaguano == 0), "sanitario1"] = 0
df_test.loc[(df_test.v14a == 1)&(df_test.sanitario1 == 1)&(df_test.abastaguano == 0), "v14a"] = 0
df_test.loc[(df_test.v14a == 1)&(df_test.sanitario1 == 1)&(df_test.abastaguano == 0), "sanitario1"] = 0
testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=0,
pin_memory=True
)<define_variables>
df_train['rez_esc'].fillna(0, inplace=True)
 df_test['rez_esc'].fillna(0, inplace=True )
threshold_label = [0.50,0.50,0.50,0.50,]<define_variables>
df_train['SQBmeaned'].fillna(0, inplace=True)
 df_test['SQBmeaned'].fillna(0, inplace=True )
augment = ['null']<prepare_output>
df_train['meaneduc'].fillna(0, inplace=True)
 df_test['meaneduc'].fillna(0, inplace=True )
probability_label, image_id = get_classification_preds(model_classification, testset)
 predict_label = probability_label>np.array(threshold_label ).reshape(1,4,1,1)
 image_id_class_id = []
encoded_pixel = []
for b in range(len(image_id)) :
for c in range(4):
image_id_class_id.append(image_id[b]+'_%d'%(c+1))
if predict_label[b,c]==0:
rle=''
else:
rle ='1 1'
encoded_pixel.append(rle)
 df_classification = pd.DataFrame(zip(image_id_class_id, encoded_pixel), columns=['ImageId_ClassId', 'EncodedPixels'] )<set_options>
df_train['v18q'].value_counts()
warnings.filterwarnings('ignore')
<load_pretrained>
df_train.loc[df_train['v18q'] == 1, 'v18q1'].value_counts()
unet_se_resnext50_32x4d = \
load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
unet_resnet34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()<normalization>
df_train.loc[df_train['v18q'] == 0, 'v18q1'].value_counts()
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = Model([unet_se_resnext50_32x4d, unet_mobilenet2, unet_resnet34] )<categorify>
df_train['v18q1'].fillna(0, inplace=True)
 df_test['v18q1'].fillna(0, inplace=True )
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.230, 0.225, 0.223)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 2
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]<define_search_space>
df_train['tipovivi3'].value_counts()
thresholds = [0.5, 0.5, 0.5, 0.5]
min_area = [600, 600, 1000, 2000]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
image_file = []
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
p = datasets[i].inverse(p)
 preds.append(p)
 image_file = batch['image_file']
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
for p, file in zip(preds, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
<data_type_conversions>
df_train['v2a1'].fillna(0, inplace=True)
 df_test['v2a1'].fillna(0, inplace=True )
df = pd.DataFrame(res)
 df = df.fillna('')
 df['Class'] = df['ImageId_ClassId'].str[-1].astype(np.int32)
 df['Label'] =(df['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df['Class']==1)&(df['Label']==1)).sum()
pos2 =(( df['Class']==2)&(df['Label']==1)).sum()
pos3 =(( df['Class']==3)&(df['Label']==1)).sum()
pos4 =(( df['Class']==4)&(df['Label']==1)).sum()
num_image = len(df)//4
num = len(df)
 pos =(df['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))<feature_engineering>
total = df_train.isnull().sum().sort_values(ascending=False)
 percent = 100 *(df_train.isnull().sum() / df_train.isnull().count() ).sort_values(ascending=False)
 missing_df = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_df.head(20 )
df_mask = pd.DataFrame(res)
 df_mask = df_mask.fillna('')
 df_label = df_classification.copy()
assert(np.all(df_mask['ImageId_ClassId'].values == df_label['ImageId_ClassId'].values))
print(( df_mask.loc[df_label['EncodedPixels']=='','EncodedPixels'] != '' ).sum())
 df_mask.loc[df_label['EncodedPixels']=='','EncodedPixels']=''<save_to_csv>
total = df_test.isnull().sum().sort_values(ascending=False)
 percent = 100 *(df_test.isnull().sum() / df_test.isnull().count() ).sort_values(ascending=False)
 missing_df = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_df.head(20 )
df_mask.to_csv("submission.csv", index=False )<data_type_conversions>
features_object = [col for col in df_train.columns if df_train[col].dtype == 'object']
df_mask['Class'] = df_mask['ImageId_ClassId'].str[-1].astype(np.int32)
 df_mask['Label'] =(df_mask['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df_mask['Class']==1)&(df_mask['Label']==1)).sum()
pos2 =(( df_mask['Class']==2)&(df_mask['Label']==1)).sum()
pos3 =(( df_mask['Class']==3)&(df_mask['Label']==1)).sum()
pos4 =(( df_mask['Class']==4)&(df_mask['Label']==1)).sum()
num_image = len(df_mask)//4
num = len(df_mask)
 pos =(df_mask['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))<set_options>
df_train['dependency'] = np.sqrt(df_train['SQBdependency'])
 df_test['dependency'] = np.sqrt(df_test['SQBdependency'] )
warnings.filterwarnings('ignore')
<load_pretrained>
eff_net = load('.. /input/severstaleffnet/effnetb7_combined.pth' ).cuda() 
<define_variables>
def replace_edjefe(x):
if x == 'yes':
return 1
elif x == 'no':
return 0
else:
return x
df_train['edjefe'] = df_train['edjefe'].apply(replace_edjefe ).astype(float)
 df_test['edjefe'] = df_test['edjefe'].apply(replace_edjefe ).astype(float )
__all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',
'se_resnext50_32x4d', 'se_resnext101_32x4d']
pretrained_settings = {
'senet154': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet50': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet101': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet152': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext50_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext101_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
}
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def initialize_pretrained_model(model, num_classes, settings):
assert num_classes == settings['num_classes'], \
'num_classes should be {}, but is {}'.format(
 settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
def senet154(num_classes=1000, pretrained='imagenet'):
model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,
dropout_p=0.2, num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['senet154'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet50(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet50'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet101(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet101'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet152(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet152'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext50_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext101_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
class SENetEncoder(SENet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.last_linear
del self.avg_pool
def forward(self, x):
for module in self.layer0[:-1]:
x = module(x)
 x0 = x
x = self.layer0[-1](x)
 x1 = self.layer1(x)
 x2 = self.layer2(x1)
 x3 = self.layer3(x2)
 x4 = self.layer4(x3)
 features = [x4, x3, x2, x1, x0]
return features
def load_state_dict(self, state_dict, **kwargs):
state_dict.pop('last_linear.bias')
 state_dict.pop('last_linear.weight')
 super().load_state_dict(state_dict, **kwargs)
 senet_encoders = {
'senet154': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 128),
'params': {
'block': SEBottleneck,
'dropout_p': 0.2,
'groups': 64,
'layers': [3, 8, 36, 3],
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet50': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 4, 6, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet101': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 4, 23, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet152': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 8, 36, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnext50_32x4d': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNeXtBottleneck,
'layers': [3, 4, 6, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 32,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnext101_32x4d': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNeXtBottleneck,
'layers': [3, 4, 23, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 32,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
}
class ResNetEncoder(ResNet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.fc
def forward(self, x):
x0 = self.conv1(x)
 x0 = self.bn1(x0)
 x0 = self.relu(x0)
 x1 = self.maxpool(x0)
 x1 = self.layer1(x1)
 x2 = self.layer2(x1)
 x3 = self.layer3(x2)
 x4 = self.layer4(x3)
 return [x4, x3, x2, x1, x0]
def load_state_dict(self, state_dict, **kwargs):
state_dict.pop('fc.bias')
 state_dict.pop('fc.weight')
 super().load_state_dict(state_dict, **kwargs)
 resnet_encoders = {
'resnet18': {
'encoder': ResNetEncoder,
'out_shapes':(512, 256, 128, 64, 64),
'params': {
'block': BasicBlock,
'layers': [2, 2, 2, 2],
},
},
'resnet34': {
'encoder': ResNetEncoder,
'out_shapes':(512, 256, 128, 64, 64),
'params': {
'block': BasicBlock,
'layers': [3, 4, 6, 3],
},
},
'resnet50': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 6, 3],
},
},
'resnet101': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 23, 3],
},
},
'resnet152': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 8, 36, 3],
},
},
}
class DenseNetEncoder(DenseNet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.classifier
self.initialize()
@staticmethod
def _transition(x, transition_block):
for module in transition_block:
x = module(x)
 if isinstance(module, nn.ReLU):
skip = x
return x, skip
def initialize(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 def forward(self, x):
x = self.features.conv0(x)
 x = self.features.norm0(x)
 x = self.features.relu0(x)
 x0 = x
x = self.features.pool0(x)
 x = self.features.denseblock1(x)
 x, x1 = self._transition(x, self.features.transition1)
 x = self.features.denseblock2(x)
 x, x2 = self._transition(x, self.features.transition2)
 x = self.features.denseblock3(x)
 x, x3 = self._transition(x, self.features.transition3)
 x = self.features.denseblock4(x)
 x4 = self.features.norm5(x)
 features = [x4, x3, x2, x1, x0]
return features
def load_state_dict(self, state_dict):
pattern = re.compile(
 r'^ (.*denselayer\d+\.( ?:norm|relu|conv)) \.((?:[12])\.( ?:weight|bias|running_mean|running_var)) $')
 for key in list(state_dict.keys()):
res = pattern.match(key)
 if res:
new_key = res.group(1)+ res.group(2)
 state_dict[new_key] = state_dict[key]
del state_dict[key]
state_dict.pop('classifier.bias')
 state_dict.pop('classifier.weight')
 super().load_state_dict(state_dict)
 densenet_encoders = {
'densenet121': {
'encoder': DenseNetEncoder,
'out_shapes':(1024, 1024, 512, 256, 64),
'params': {
'num_init_features': 64,
'growth_rate': 32,
'block_config':(6, 12, 24, 16),
}
},
'densenet169': {
'encoder': DenseNetEncoder,
'out_shapes':(1664, 1280, 512, 256, 64),
'params': {
'num_init_features': 64,
'growth_rate': 32,
'block_config':(6, 12, 32, 32),
}
},
'densenet201': {
'encoder': DenseNetEncoder,
'out_shapes':(1920, 1792, 512, 256, 64),
'params': {
'num_init_features': 64,
'growth_rate': 32,
'block_config':(6, 12, 48, 32),
}
},
'densenet161': {
'encoder': DenseNetEncoder,
'out_shapes':(2208, 2112, 768, 384, 96),
'params': {
'num_init_features': 96,
'growth_rate': 48,
'block_config':(6, 12, 36, 24),
}
},
}
encoders = {}
encoders.update(resnet_encoders)
 encoders.update(senet_encoders)
 encoders.update(densenet_encoders)
 def get_encoder(name, encoder_weights=None):
Encoder = encoders[name]['encoder']
encoder = Encoder(**encoders[name]['params'])
 encoder.out_shapes = encoders[name]['out_shapes']
if encoder_weights is not None:
settings = encoders[name]['pretrained_settings'][encoder_weights]
encoder.load_state_dict(model_zoo.load_url(settings['url']))
return encoder
def get_encoder_names() :
return list(encoders.keys())
 def get_preprocessing_params(encoder_name, pretrained='imagenet'):
settings = encoders[encoder_name]['pretrained_settings']
if pretrained not in settings.keys() :
raise ValueError('Avaliable pretrained options {}'.format(settings.keys()))
formatted_settings = {}
formatted_settings['input_space'] = settings[pretrained].get('input_space')
 formatted_settings['input_range'] = settings[pretrained].get('input_range')
 formatted_settings['mean'] = settings[pretrained].get('mean')
 formatted_settings['std'] = settings[pretrained].get('std')
 return formatted_settings
def get_preprocessing_fn(encoder_name, pretrained='imagenet'):
params = get_preprocessing_params(encoder_name, pretrained=pretrained)
 return functools.partial(preprocess_input, **params)
 class Model(nn.Module):
def __init__(self):
super().__init__()
def initialize(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 class EncoderDecoder(Model):
def __init__(self, encoder, decoder, activation):
super().__init__()
self.encoder = encoder
self.decoder = decoder
if callable(activation)or activation is None:
self.activation = activation
elif activation == 'softmax':
self.activation = nn.Softmax(dim=1)
 elif activation == 'sigmoid':
self.activation = nn.Sigmoid()
else:
raise ValueError('Activation should be "sigmoid"/"softmax"/callable/None')
 def forward(self, x):

 x = self.encoder(x)
 x = self.decoder(x)
 return x
def predict(self, x):

 if self.training:
self.eval()
with torch.no_grad() :
x = self.forward(x)
 if self.activation:
x = self.activation(x)
 return x
class DecoderBlock(nn.Module):
def __init__(self, in_channels, out_channels, use_batchnorm=True, attention_type=None):
super().__init__()
if attention_type is None:
self.attention1 = nn.Identity()
self.attention2 = nn.Identity()
elif attention_type == 'scse':
self.attention1 = SCSEModule(in_channels)
 self.attention2 = SCSEModule(out_channels)
 self.block = nn.Sequential(
 Conv2dReLU(in_channels, out_channels, kernel_size=3, padding=1, use_batchnorm=use_batchnorm),
Conv2dReLU(out_channels, out_channels, kernel_size=3, padding=1, use_batchnorm=use_batchnorm), 
)
 def forward(self, x):
x, skip = x
x = F.interpolate(x, scale_factor=2, mode='nearest')
 if skip is not None:
x = torch.cat([x, skip], dim=1)
 x = self.attention1(x)
 x = self.block(x)
 x = self.attention2(x)
 return x
class CenterBlock(DecoderBlock):
def forward(self, x):
return self.block(x)
 class UnetDecoder(Model):
def __init__(
 self,
encoder_channels,
decoder_channels=(256, 128, 64, 32, 16),
final_channels=1,
use_batchnorm=True,
center=False,
attention_type=None 
):
super().__init__()
if center:
channels = encoder_channels[0]
self.center = CenterBlock(channels, channels, use_batchnorm=use_batchnorm)
 else:
self.center = None
in_channels = self.compute_channels(encoder_channels, decoder_channels)
 out_channels = decoder_channels
self.layer1 = DecoderBlock(in_channels[0], out_channels[0],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer2 = DecoderBlock(in_channels[1], out_channels[1],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer3 = DecoderBlock(in_channels[2], out_channels[2],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer4 = DecoderBlock(in_channels[3], out_channels[3],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer5 = DecoderBlock(in_channels[4], out_channels[4],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.final_conv = nn.Conv2d(out_channels[4], final_channels, kernel_size=(1, 1))
self.initialize()
def compute_channels(self, encoder_channels, decoder_channels):
channels = [
encoder_channels[0] + encoder_channels[1],
encoder_channels[2] + decoder_channels[0],
encoder_channels[3] + decoder_channels[1],
encoder_channels[4] + decoder_channels[2],
0 + decoder_channels[3],
]
return channels
def forward(self, x):
encoder_head = x[0]
skips = x[1:]
if self.center:
encoder_head = self.center(encoder_head)
 x = self.layer1([encoder_head, skips[0]])
 x = self.layer2([x, skips[1]])
 x = self.layer3([x, skips[2]])
 x = self.layer4([x, skips[3]])
 x = self.layer5([x, None])
 x = self.final_conv(x)
 return x
class Conv2dReLU(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size, padding=0,
stride=1, use_batchnorm=True, **batchnorm_params):
super().__init__()
layers = [
nn.Conv2d(in_channels, out_channels, kernel_size,
stride=stride, padding=padding, bias=not(use_batchnorm)) ,
nn.ReLU(inplace=True),
]
if use_batchnorm:
layers.insert(1, nn.BatchNorm2d(out_channels, **batchnorm_params))
self.block = nn.Sequential(*layers)
 def forward(self, x):
return self.block(x)
 class Unet(EncoderDecoder):

 def __init__(
 self,
encoder_name='resnet34',
encoder_weights='imagenet',
decoder_use_batchnorm=True,
decoder_channels=(256, 128, 64, 32, 16),
classes=1,
activation='sigmoid',
center=False,
attention_type=None 
):
encoder = get_encoder(
 encoder_name,
encoder_weights=encoder_weights 
)
 decoder = UnetDecoder(
 encoder_channels=encoder.out_shapes,
decoder_channels=decoder_channels,
final_channels=classes,
use_batchnorm=decoder_use_batchnorm,
center=center,
attention_type=attention_type 
)
 super().__init__(encoder, decoder, activation)
 self.name = 'u-{}'.format(encoder_name )<train_model>
def replace_edjefa(x):
if x == 'yes':
return 1
elif x == 'no':
return 0
else:
return x
df_train['edjefa'] = df_train['edjefa'].apply(replace_edjefa ).astype(float)
 df_test['edjefa'] = df_test['edjefa'].apply(replace_edjefa ).astype(float )
unet2 = Unet('se_resnext50_32x4d', classes=4, encoder_weights=None, activation='softmax')
 ckpt_path = '.. /input/weight-segmentation/se_resnext50_32x4d_Unet_checkpoint_185.pth'
device = torch.device('cuda')
 unet2 = unet2.to(device)
 unet2.eval()
print() 
<normalization>
df_train['edjef'] = np.max(df_train[['edjefa','edjefe']], axis=1)
 df_test['edjef'] = np.max(df_test[['edjefa','edjefe']], axis=1 )
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = eff_net<categorify>
df_train['roof_waste_material'] = np.nan
df_test['roof_waste_material'] = np.nan
df_train['electricity_other'] = np.nan
df_test['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
df_train['roof_waste_material'] = df_train.apply(lambda x : fill_roof_exception(x),axis=1)
 df_test['roof_waste_material'] = df_test.apply(lambda x : fill_roof_exception(x),axis=1)
 df_train['electricity_other'] = df_train.apply(lambda x : fill_no_electricity(x),axis=1)
 df_test['electricity_other'] = df_test.apply(lambda x : fill_no_electricity(x),axis=1 )
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 1
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)],
[A.VerticalFlip(p=1)],
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]<save_to_csv>
binary_cat_features = [col for col in df_train.columns if df_train[col].value_counts().shape[0] == 2]
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )<feature_engineering>
continuous_features = [col for col in df_train.columns if col not in binary_cat_features]
continuous_features = [col for col in continuous_features if col not in features_object]
continuous_features = [col for col in continuous_features if col not in ['Id', 'Target', 'idhogar']]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()<install_modules>
print('There are {} continuous features'.format(len(continuous_features)))
 for col in continuous_features:
print('{}: {}'.format(col, description.loc[description['varname'] == col, 'description'].values))
!pip install --no-deps.. /input/segmentations-models-pytorch-7fa1020/pretrainedmodels-0.7.4-py3-none-any.whl
sys.path.append('.. /input/smp-latest/segmentation_models.pytorch-master/')
 sys.path.append('.. /input/efficientnet-pytorch/efficientnet_pytorch-0.4.0/' )<load_from_csv>
df_train['edjef'].value_counts()
<load_from_csv>
df_train.drop('tamhog', axis=1, inplace=True)
 df_test.drop('tamhog', axis=1, inplace=True )
df=pd.read_csv('.. /input/steel-classifier-csv/train_classes.csv')
 df.Label =(df.Label!='0' ).astype('int')
 sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_df= pd.read_csv(sample_submission_path)
 def changename(x):
x=x[:-2]
return x
test_df.ImageId_ClassId=test_df.ImageId_ClassId.apply(changename)
 test_df=test_df.drop_duplicates(['ImageId_ClassId'] )<count_values>
df_train['adult'] = df_train['hogar_adul'] - df_train['hogar_mayor']
df_train['dependency_count'] = df_train['hogar_nin'] + df_train['hogar_mayor']
df_train['dependency'] = df_train['dependency_count'] / df_train['adult']
df_train['child_percent'] = df_train['hogar_nin'] / df_train['hogar_total']
df_train['elder_percent'] = df_train['hogar_mayor'] / df_train['hogar_total']
df_train['adult_percent'] = df_train['hogar_adul'] / df_train['hogar_total']
df_train['males_younger_12_years_percent'] = df_train['r4h1'] / df_train['hogar_total']
df_train['males_older_12_years_percent'] = df_train['r4h2'] / df_train['hogar_total']
df_train['males_percent'] = df_train['r4h3'] / df_train['hogar_total']
df_train['females_younger_12_years_percent'] = df_train['r4m1'] / df_train['hogar_total']
df_train['females_older_12_years_percent'] = df_train['r4m2'] / df_train['hogar_total']
df_train['females_percent'] = df_train['r4m3'] / df_train['hogar_total']
df_train['persons_younger_12_years_percent'] = df_train['r4t1'] / df_train['hogar_total']
df_train['persons_older_12_years_percent'] = df_train['r4t2'] / df_train['hogar_total']
df_train['persons_percent'] = df_train['r4t3'] / df_train['hogar_total']
test_df.EncodedPixels.value_counts()<import_modules>
df_test['adult'] = df_test['hogar_adul'] - df_test['hogar_mayor']
df_test['dependency_count'] = df_test['hogar_nin'] + df_test['hogar_mayor']
df_test['dependency'] = df_test['dependency_count'] / df_test['adult']
df_test['child_percent'] = df_test['hogar_nin'] / df_test['hogar_total']
df_test['elder_percent'] = df_test['hogar_mayor'] / df_test['hogar_total']
df_test['adult_percent'] = df_test['hogar_adul'] / df_test['hogar_total']
df_test['males_younger_12_years_percent'] = df_test['r4h1'] / df_test['hogar_total']
df_test['males_older_12_years_percent'] = df_test['r4h2'] / df_test['hogar_total']
df_test['males_percent'] = df_test['r4h3'] / df_test['hogar_total']
df_test['females_younger_12_years_percent'] = df_test['r4m1'] / df_test['hogar_total']
df_test['females_older_12_years_percent'] = df_test['r4m2'] / df_test['hogar_total']
df_test['females_percent'] = df_test['r4m3'] / df_test['hogar_total']
df_test['persons_younger_12_years_percent'] = df_test['r4t1'] / df_test['hogar_total']
df_test['persons_older_12_years_percent'] = df_test['r4t2'] / df_test['hogar_total']
df_test['persons_percent'] = df_test['r4t3'] / df_test['hogar_total']
import pdb
import os
import cv2
import pandas as pd
import numpy as np
from tqdm import tqdm_notebook as tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose, HorizontalFlip)
 from albumentations.pytorch import ToTensor
import torch.utils.data as data<set_options>
df_train['males_younger_12_years_in_household_size'] = df_train['r4h1'] / df_train['hhsize']
df_train['males_older_12_years_in_household_size'] = df_train['r4h2'] / df_train['hhsize']
df_train['males_in_household_size'] = df_train['r4h3'] / df_train['hhsize']
df_train['females_younger_12_years_in_household_size'] = df_train['r4m1'] / df_train['hhsize']
df_train['females_older_12_years_in_household_size'] = df_train['r4m2'] / df_train['hhsize']
df_train['females_in_household_size'] = df_train['r4m3'] / df_train['hogar_total']
df_train['persons_younger_12_years_in_household_size'] = df_train['r4t1'] / df_train['hhsize']
df_train['persons_older_12_years_in_household_size'] = df_train['r4t2'] / df_train['hhsize']
df_train['persons_in_household_size'] = df_train['r4t3'] / df_train['hhsize']
device = torch.device("cuda" )<categorify>
df_test['males_younger_12_years_in_household_size'] = df_test['r4h1'] / df_test['hhsize']
df_test['males_older_12_years_in_household_size'] = df_test['r4h2'] / df_test['hhsize']
df_test['males_in_household_size'] = df_test['r4h3'] / df_test['hhsize']
df_test['females_younger_12_years_in_household_size'] = df_test['r4m1'] / df_test['hhsize']
df_test['females_older_12_years_in_household_size'] = df_test['r4m2'] / df_test['hhsize']
df_test['females_in_household_size'] = df_test['r4m3'] / df_test['hogar_total']
df_test['persons_younger_12_years_in_household_size'] = df_test['r4t1'] / df_test['hhsize']
df_test['persons_older_12_years_in_household_size'] = df_test['r4t2'] / df_test['hhsize']
df_test['persons_in_household_size'] = df_test['r4t3'] / df_test['hhsize']
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<feature_engineering>
df_train['overcrowding_room_and_bedroom'] =(df_train['hacdor'] + df_train['hacapo'])/2
df_test['overcrowding_room_and_bedroom'] =(df_test['hacdor'] + df_test['hacapo'])/2
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor()
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<categorify>
df_train['escolari_age'] = df_train['escolari']/df_train['age']
df_test['escolari_age'] = df_test['escolari']/df_test['age']
df_train['age_12_19'] = df_train['hogar_nin'] - df_train['r4t1']
df_test['age_12_19'] = df_test['hogar_nin'] - df_test['r4t1']
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<define_variables>
df_train['phones-per-capita'] = df_train['qmobilephone'] / df_train['tamviv']
df_train['tablets-per-capita'] = df_train['v18q1'] / df_train['tamviv']
df_train['rooms-per-capita'] = df_train['rooms'] / df_train['tamviv']
df_train['rent-per-capita'] = df_train['v2a1'] / df_train['tamviv']
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"<load_from_csv>
df_test['phones-per-capita'] = df_test['qmobilephone'] / df_test['tamviv']
df_test['tablets-per-capita'] = df_test['v18q1'] / df_test['tamviv']
df_test['rooms-per-capita'] = df_test['rooms'] / df_test['tamviv']
df_test['rent-per-capita'] = df_test['v2a1'] / df_test['tamviv']
test_df_= pd.read_csv(sample_submission_path )<load_from_csv>
family_size_features = ['adult', 'hogar_adul', 'hogar_mayor', 'hogar_nin', 'hogar_total', 'r4h1',
'r4h2', 'r4h3', 'r4m1', 'r4m2', 'r4m3', 'r4t1', 'r4t2', 'r4t3', 'hhsize']
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('v2a1', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['v2a1'] / df_train[col]
df_test[new_col_name] = df_test['v2a1'] / df_test[col]
fold_csv = pd.read_csv(sample_submission_path)
 num_workers = 4
batch_size = 4
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, test_df_, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)<set_options>
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )
GlobalParams = namedtuple('GlobalParams', ['batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate', 'num_classes',
'width_coefficient', 'depth_coefficient', 'depth_divisor', 'min_depth',
'drop_connect_rate'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs = namedtuple('BlockArgs', ['kernel_size', 'num_repeat', 'input_filters', 'output_filters', 'expand_ratio',
'id_skip', 'strides', 'se_ratio'])
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 IMAGENET_WEIGHTS = {
'efficientnet-b0': 'http://storage.googleapis.com/public-models/efficientnet-b0-08094119.pth',
'efficientnet-b1': 'http://storage.googleapis.com/public-models/efficientnet-b1-dbc7070a.pth',
'efficientnet-b2': 'http://storage.googleapis.com/public-models/efficientnet-b2-27687264.pth',
'efficientnet-b3': 'http://storage.googleapis.com/public-models/efficientnet-b3-c8376fa2.pth',
'efficientnet-b4': 'http://storage.googleapis.com/public-models/efficientnet-b4-e116e8b3.pth',
'efficientnet-b5': 'http://storage.googleapis.com/public-models/efficientnet-b5-586e6cc6.pth'
}
def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
if not multiplier:
return filters
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def get_efficientnet_params(model_name, override_params=None):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
}
if model_name not in params_dict.keys() :
raise KeyError('There is no model named {}.'.format(model_name))
width_coefficient, depth_coefficient, _, dropout_rate = params_dict[model_name]
blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=0.2,
num_classes=1000,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None)
 if override_params:
global_params = global_params._replace(**override_params)
 decoder = BlockDecoder()
return decoder.decode(blocks_args), global_params
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
if 's' not in options or len(options['s'])!= 2:
raise ValueError('Strides options should be a pair of integers.')
 return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
strides=[int(options['s'][0]), int(options['s'][1])] 
)
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 def decode(self, string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(self._decode_block_string(block_string))
return blocks_args
def encode(self, blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(self._encode_block_string(block))
return block_strings
class EfficientNet(nn.Module):
def __init__(self, block_args_list, global_params):
super().__init__()
self.block_args_list = block_args_list
self.global_params = global_params
batch_norm_momentum = 1 - self.global_params.batch_norm_momentum
batch_norm_epsilon = self.global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(32, self.global_params)
 self._conv_stem = Conv2dSamePadding(in_channels,
out_channels,
kernel_size=3,
stride=2,
bias=False,
name='stem_conv')
 self._bn0 = BatchNorm2d(num_features=out_channels,
momentum=batch_norm_momentum,
eps=batch_norm_epsilon,
name='stem_batch_norm')
 self._swish = Swish(name='swish')
 idx = 0
self._blocks = nn.ModuleList([])
 for block_args in self.block_args_list:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self.global_params),
output_filters=round_filters(block_args.output_filters, self.global_params),
num_repeat=round_repeats(block_args.num_repeat, self.global_params)
)
 self._blocks.append(MBConvBlock(block_args, self.global_params, idx=idx))
idx += 1
if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, strides=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self.global_params, idx=idx))
idx += 1
in_channels = block_args.output_filters
out_channels = round_filters(1280, self.global_params)
 self._conv_head = Conv2dSamePadding(in_channels,
out_channels,
kernel_size=1,
bias=False,
name='head_conv')
 self._bn1 = BatchNorm2d(num_features=out_channels,
momentum=batch_norm_momentum,
eps=batch_norm_epsilon,
name='head_batch_norm')
 self.dropout_rate = self.global_params.dropout_rate
self._fc = nn.Linear(out_channels, self.global_params.num_classes)
 def forward(self, x):
x = self._conv_stem(x)
 x = self._bn0(x)
 x = self._swish(x)
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self.global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= idx / len(self._blocks)
 x = block(x, drop_connect_rate)
 x = self._conv_head(x)
 x = self._bn1(x)
 x = self._swish(x)
 x = F.adaptive_avg_pool2d(x, 1 ).squeeze(-1 ).squeeze(-1)
 if self.dropout_rate > 0:
x = F.dropout(x, p=self.dropout_rate, training=self.training)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, *, n_classes=1000, pretrained=False):
return _get_model_by_name(model_name, classes=n_classes, pretrained=pretrained)
 @classmethod
def encoder(cls, model_name, *, pretrained=False):
model = cls.from_name(model_name, pretrained=pretrained)
 class Encoder(nn.Module):
def __init__(self):
super().__init__()
self.name = model_name
self.global_params = model.global_params
self.stem_conv = model._conv_stem
self.stem_batch_norm = model._bn0
self.stem_swish = Swish(name='stem_swish')
 self.blocks = model._blocks
self.head_conv = model._conv_head
self.head_batch_norm = model._bn1
self.head_swish = Swish(name='head_swish')
 def forward(self, x):
x = self.stem_conv(x)
 x = self.stem_batch_norm(x)
 x = self.stem_swish(x)
 for idx, block in enumerate(self.blocks):
drop_connect_rate = self.global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= idx / len(self.blocks)
 x = block(x, drop_connect_rate)
 x = self.head_conv(x)
 x = self.head_batch_norm(x)
 x = self.head_swish(x)
 return x
return Encoder()
@classmethod
def custom_head(cls, model_name, *, n_classes=1000, pretrained=False):
if n_classes == 1000:
return cls.from_name(model_name, n_classes=n_classes, pretrained=pretrained)
 else:
class CustomHead(nn.Module):
def __init__(self, out_channels):
super().__init__()
self.encoder = cls.encoder(model_name, pretrained=pretrained)
 self.custom_head = custom_head(self.n_channels * 2, out_channels)
 @property
def n_channels(self):
n_channels_dict = {'efficientnet-b0': 1280, 'efficientnet-b1': 1280, 'efficientnet-b2': 1408,
'efficientnet-b3': 1536, 'efficientnet-b4': 1792, 'efficientnet-b5': 2048,
'efficientnet-b6': 2304, 'efficientnet-b7': 2560}
return n_channels_dict[self.encoder.name]
def forward(self, x):
x = self.encoder(x)
 mp = nn.AdaptiveMaxPool2d(output_size=(1, 1))(x)
 ap = nn.AdaptiveAvgPool2d(output_size=(1, 1))(x)
 x = torch.cat([mp, ap], dim=1)
 x = x.view(x.size(0), -1)
 x = self.custom_head(x)
 return x
return CustomHead(n_classes)
 def _get_model_by_name(model_name, classes=1000, pretrained=False):
block_args_list, global_params = get_efficientnet_params(model_name, override_params={'num_classes': classes})
 model = EfficientNet(block_args_list, global_params)
 try:
if pretrained:
pretrained_state_dict = load_state_dict_from_url(IMAGENET_WEIGHTS[model_name])
 if classes != 1000:
random_state_dict = model.state_dict()
pretrained_state_dict['_fc.weight'] = random_state_dict['_fc.weight']
pretrained_state_dict['_fc.bias'] = random_state_dict['_fc.bias']
model.load_state_dict(pretrained_state_dict)
 except KeyError as e:
print(f"NOTE: Currently model {e} doesn't have pretrained weights, therefore a model with randomly initialized"
" weights is returned.")
 return model
class Swish(nn.Module):
def __init__(self, name=None):
super().__init__()
self.name = name
def forward(self, x):
return x * torch.sigmoid(x)
 class Conv2dSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True, name=None):
super().__init__(in_channels, out_channels, kernel_size, stride, padding=0, dilation=dilation, groups=groups,
bias=bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
self.name = name
def forward(self, x):
input_h, input_w = x.size() [2:]
kernel_h, kernel_w = self.weight.size() [2:]
stride_h, stride_w = self.stride
output_h, output_w = math.ceil(input_h / stride_h), math.ceil(input_w / stride_w)
 pad_h = max(( output_h - 1)* self.stride[0] +(kernel_h - 1)* self.dilation[0] + 1 - input_h, 0)
 pad_w = max(( output_w - 1)* self.stride[1] +(kernel_w - 1)* self.dilation[1] + 1 - input_w, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class BatchNorm2d(nn.BatchNorm2d):
def __init__(self, num_features, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True, name=None):
super().__init__(num_features, eps=eps, momentum=momentum, affine=affine,
track_running_stats=track_running_stats)
 self.name = name
def drop_connect(inputs, drop_connect_rate, training):
if not training:
return inputs
batch_size = inputs.shape[0]
keep_prob = 1.0 - drop_connect_rate
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params, idx):
super().__init__()
block_name = 'blocks_' + str(idx)+ '_'
self.block_args = block_args
self.batch_norm_momentum = 1 - global_params.batch_norm_momentum
self.batch_norm_epsilon = global_params.batch_norm_epsilon
self.has_se =(self.block_args.se_ratio is not None)and(0 < self.block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
self.swish = Swish(block_name + '_swish')
 in_channels = self.block_args.input_filters
out_channels = self.block_args.input_filters * self.block_args.expand_ratio
if self.block_args.expand_ratio != 1:
self._expand_conv = Conv2dSamePadding(in_channels=in_channels,
out_channels=out_channels,
kernel_size=1,
bias=False,
name=block_name + 'expansion_conv')
 self._bn0 = BatchNorm2d(num_features=out_channels,
momentum=self.batch_norm_momentum,
eps=self.batch_norm_epsilon,
name=block_name + 'expansion_batch_norm')
 kernel_size = self.block_args.kernel_size
strides = self.block_args.strides
self._depthwise_conv = Conv2dSamePadding(in_channels=out_channels,
out_channels=out_channels,
groups=out_channels,
kernel_size=kernel_size,
stride=strides,
bias=False,
name=block_name + 'depthwise_conv')
 self._bn1 = BatchNorm2d(num_features=out_channels,
momentum=self.batch_norm_momentum,
eps=self.batch_norm_epsilon,
name=block_name + 'depthwise_batch_norm')
 if self.has_se:
num_squeezed_channels = max(1, int(self.block_args.input_filters * self.block_args.se_ratio))
self._se_reduce = Conv2dSamePadding(in_channels=out_channels,
out_channels=num_squeezed_channels,
kernel_size=1,
name=block_name + 'se_reduce')
 self._se_expand = Conv2dSamePadding(in_channels=num_squeezed_channels,
out_channels=out_channels,
kernel_size=1,
name=block_name + 'se_expand')
 final_output_channels = self.block_args.output_filters
self._project_conv = Conv2dSamePadding(in_channels=out_channels,
out_channels=final_output_channels,
kernel_size=1,
bias=False,
name=block_name + 'output_conv')
 self._bn2 = BatchNorm2d(num_features=final_output_channels,
momentum=self.batch_norm_momentum,
eps=self.batch_norm_epsilon,
name=block_name + 'output_batch_norm')
 def forward(self, x, drop_connect_rate=None):
identity = x
if self.block_args.expand_ratio != 1:
x = self._expand_conv(x)
 x = self._bn0(x)
 x = self.swish(x)
 x = self._depthwise_conv(x)
 x = self._bn1(x)
 x = self.swish(x)
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(self.swish(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self.block_args.input_filters, self.block_args.output_filters
if self.id_skip and self.block_args.strides == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, drop_connect_rate=drop_connect_rate, training=self.training)
 x = x + identity
return x
def double_conv(in_channels, out_channels):
return nn.Sequential(
 nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True),
nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True)
)
 def up_conv(in_channels, out_channels):
return nn.ConvTranspose2d(
 in_channels, out_channels, kernel_size=2, stride=2 
)
 def custom_head(in_channels, out_channels):
return nn.Sequential(
 nn.Dropout() ,
nn.Linear(in_channels, 512),
nn.ReLU(inplace=True),
nn.Dropout() ,
nn.Linear(512, out_channels)
)
 __all__ = ['EfficientUnet', 'get_efficientunet_b0', 'get_efficientunet_b1', 'get_efficientunet_b2',
'get_efficientunet_b3', 'get_efficientunet_b4', 'get_efficientunet_b5', 'get_efficientunet_b6',
'get_efficientunet_b7']
def get_blocks_to_be_concat(model, x):
shapes = set()
blocks = OrderedDict()
hooks = []
count = 0
def register_hook(module):
def hook(module, input, output):
try:
nonlocal count
if module.name == f'blocks_{count}_output_batch_norm':
count += 1
shape = output.size() [-2:]
if shape not in shapes:
shapes.add(shape)
 blocks[module.name] = output
elif module.name == 'head_swish':
blocks.popitem()
blocks[module.name] = output
except AttributeError:
pass
if(
 not isinstance(module, nn.Sequential)
 and not isinstance(module, nn.ModuleList)
 and not(module == model)
):
hooks.append(module.register_forward_hook(hook))
model.apply(register_hook)
 model(x)
 for h in hooks:
h.remove()
return blocks
class Attention_block(nn.Module):
def __init__(self,F_g,F_l,F_int):
super(Attention_block,self ).__init__()
self.W_g = nn.Sequential(
 nn.Conv2d(F_g, F_int, kernel_size=1,stride=1,padding=0,bias=True),
nn.BatchNorm2d(F_int)
)
 self.W_xa = nn.Sequential(
 nn.Conv2d(F_l, F_int, kernel_size=1,stride=1,padding=0,bias=True),
nn.BatchNorm2d(F_int)
)
 self.psi = nn.Sequential(
 nn.Conv2d(F_int, 1, kernel_size=1,stride=1,padding=0,bias=True),
nn.BatchNorm2d(1),
nn.Sigmoid() 
)
 self.relu = nn.ReLU(inplace=True)
 def forward(self,g,xa):
g1 = self.W_g(g)
 x1 = self.W_xa(xa)
 psi = self.relu(g1+x1)
 psi = self.psi(psi)
 return xa*psi
class EfficientUnet(nn.Module):
def __init__(self, encoder, out_channels=2, concat_input=True):
super().__init__()
self.encoder = encoder
self.concat_input = concat_input
self.up_conv1 = up_conv(self.n_channels, 512)
 self.Att1 = Attention_block(F_g=512,F_l=592-512,F_int=56)
 self.double_conv1 = double_conv(self.size[0], 512)
 self.up_conv2 = up_conv(512, 256)
 self.Att2 = Attention_block(F_g=256,F_l=296-256,F_int=28)
 self.double_conv2 = double_conv(self.size[1], 256)
 self.up_conv3 = up_conv(256, 128)
 self.Att3 = Attention_block(F_g=128,F_l=152-128,F_int=16)
 self.double_conv3 = double_conv(self.size[2], 128)
 self.up_conv4 = up_conv(128, 64)
 self.Att4 = Attention_block(F_g=64,F_l=80-64,F_int=12)
 self.double_conv4 = double_conv(self.size[3], 64)
 if self.concat_input:
self.up_conv_input = up_conv(64, 32)
 self.double_conv_input = double_conv(self.size[4], 32)
 self.final_conv = nn.Conv2d(self.size[5], out_channels, kernel_size=1)
 @property
def n_channels(self):
n_channels_dict = {'efficientnet-b0': 1280, 'efficientnet-b1': 1280, 'efficientnet-b2': 1408,
'efficientnet-b3': 1536, 'efficientnet-b4': 1792, 'efficientnet-b5': 2048,
'efficientnet-b6': 2304, 'efficientnet-b7': 2560}
return n_channels_dict[self.encoder.name]
@property
def size(self):
size_dict = {'efficientnet-b0': [592, 296, 152, 80, 35, 32], 'efficientnet-b1': [592, 296, 152, 80, 35, 32],
'efficientnet-b2': [600, 304, 152, 80, 35, 32], 'efficientnet-b3': [608, 304, 160, 88, 35, 32],
'efficientnet-b4': [624, 312, 160, 88, 35, 32], 'efficientnet-b5': [640, 320, 168, 88, 35, 32],
'efficientnet-b6': [656, 328, 168, 96, 35, 32], 'efficientnet-b7': [672, 336, 176, 96, 35, 32]}
return size_dict[self.encoder.name]
def forward(self, x):
input_ = x
blocks = get_blocks_to_be_concat(self.encoder, x)
 _, x = blocks.popitem()
x1 = self.up_conv1(x)
 _, y = blocks.popitem()
x = self.Att1(g=x1,xa=y)
 x = torch.cat([x, x1], dim=1)
 x = self.double_conv1(x)
 x1 = self.up_conv2(x)
 y = blocks.popitem() [1]
x = self.Att2(g=x1,xa=y)
 x = torch.cat([x, x1], dim=1)
 x = self.double_conv2(x)
 x1 = self.up_conv3(x)
 y = blocks.popitem() [1]
x = self.Att3(g=x1,xa=y)
 x = torch.cat([x, x1], dim=1)
 x = self.double_conv3(x)
 x1 = self.up_conv4(x)
 y = blocks.popitem() [1]
x = self.Att4(g=x1,xa=y)
 x = torch.cat([x, x1], dim=1)
 x = self.double_conv4(x)
 if self.concat_input:
x = self.up_conv_input(x)
 x = torch.cat([x, input_], dim=1)
 x = self.double_conv_input(x)
 x = self.final_conv(x)
 return x
def get_efficientunet_b0(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b0', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b1(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b1', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b2(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b2', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b3(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b3', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b4(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b4', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b5(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b5', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b6(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b6', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model
def get_efficientunet_b7(out_channels=2, concat_input=True, pretrained=True):
encoder = EfficientNet.encoder('efficientnet-b7', pretrained=pretrained)
 model = EfficientUnet(encoder, out_channels=out_channels, concat_input=concat_input)
 return model 
<choose_model_class>
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('rooms', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['rooms'] / df_train[col]
df_test[new_col_name] = df_test['rooms'] / df_test[col]
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )
unet_b0 = get_efficientunet_b0(out_channels=4, concat_input=True, pretrained=False)
 unet_b1 = get_efficientunet_b1(out_channels=4, concat_input=True, pretrained=False)
 FPN_res34 = smp.FPN('resnet34', classes=4, activation=None,encoder_weights=None)
 PSP_seresx50 = smp.PSPNet('se_resnext50_32x4d', classes=4, activation=None,encoder_weights=None)
 FPN_seresx50 = smp.FPN('se_resnext50_32x4d', classes=4, activation=None,encoder_weights=None)
 FPN_b0 = smp.FPN('efficientnet-b0', classes=4, activation=None,encoder_weights=None)
 FPN_b1 = smp.FPN('efficientnet-b1', classes=4, activation=None,encoder_weights=None)
 FPN_b2 = smp.FPN('efficientnet-b2', classes=4, activation=None,encoder_weights=None)
 FPN_b3 = smp.FPN('efficientnet-b3', classes=4, activation=None,encoder_weights=None)
 FPN_b4 = smp.FPN('efficientnet-b4', classes=4, activation=None,encoder_weights=None )<load_pretrained>
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('bedrooms', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['bedrooms'] / df_train[col]
df_test[new_col_name] = df_test['bedrooms'] / df_test[col]
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )
def weight_loader(model,path):
ckpt_path = path
device = torch.device("cuda")
 model.to(device)
 model.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"])
 return model
def make_eval(models):
for i,m in enumerate(models):
models[i]=m.eval()
return models
unet_b0 = weight_loader(unet_b0,'.. /input/b0fulldata/b0att_complete.pth')
 FPN_seresx50 = weight_loader(FPN_seresx50,'.. /input/b0fulldata/fpnserex50complete.pth')
 FPN_b1 = weight_loader(FPN_b1,'.. /input/fpn-b1/fpn_b1-stage-4.pth')
 FPN_b0 = weight_loader(FPN_b0,'.. /input/b0fulldata/fpn_b0-complete-stage-2.pth')
 unet_b1 = weight_loader(unet_b1,'.. /input/classres50/effb1unetbceloss_lrreduce_full_2.pth')
 unet_se_resnext50_32x4d = torch.jit.load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda().eval()
unet_resnet34 = torch.jit.load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda().eval()
FPN_b2 = weight_loader(FPN_b2,'.. /input/ubambamodels/fpn_b2-complete.pth')
 FPN_b4 = weight_loader(FPN_b4,'.. /input/fpnb41024/fpn-b4-crop-stage-5-2-all.pth')
 FPN_b3 = weight_loader(FPN_b3,'.. /input/fpnb3stage4/fpn-b3_crop-stage-4-all.pth' )<normalization>
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('v18q1', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['v18q1'] / df_train[col]
df_test[new_col_name] = df_test['v18q1'] / df_test[col]
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )
class Model_sig:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
x = torch.flip(x,dims = [-1])
 with torch.no_grad() :
for m in self.models:
flipped_mask = m(x)
 mask = torch.flip(flipped_mask, dims = [-1])
 res.append(mask)
 res =(torch.stack(res))
return torch.mean(torch.sigmoid(res), dim=0)
 class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
x = torch.flip(x,dims = [-1])
 with torch.no_grad() :
for m in self.models:
flipped_mask = m(x)
 mask = torch.flip(flipped_mask,dims = [-1])
 res.append(mask)
 res = torch.stack(res)
 return torch.sigmoid(torch.mean(res, dim=0))
model = Model([FPN_b3,unet_b0,unet_b1, FPN_seresx50, FPN_b0, FPN_b1, FPN_b2, unet_se_resnext50_32x4d, unet_resnet34,FPN_b4])
<define_search_space>
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('qmobilephone', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['qmobilephone'] / df_train[col]
df_test[new_col_name] = df_test['qmobilephone'] / df_test[col]
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )
thresholds_max=[0.7,0.7,0.7,0.7]
thresholds_min=[0.2,0.2,0.3,0.3]
min_area=[350, 500, 750, 1000]
res = []
predictions = []
fnames_all = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
batch_preds = model(images.to(device)).detach().cpu().numpy()
for p, file in zip(batch_preds, fnames):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
p_channel_ = p_channel
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds_max[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 else:
p_channel =(p_channel_>thresholds_min[i] ).astype(np.uint8)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
 df = pd.DataFrame(res )<save_to_csv>
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('rez_esc', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['rez_esc'] / df_train[col]
df_test[new_col_name] = df_test['rez_esc'] / df_test[col]
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )
df.to_csv("submission.csv", index=False )<count_values>
df_train['rez_esc_age'] = df_train['rez_esc'] / df_train['age']
df_train['rez_esc_escolari'] = df_train['rez_esc'] / df_train['escolari']
df_test['rez_esc_age'] = df_test['rez_esc'] / df_test['age']
df_test['rez_esc_escolari'] = df_test['rez_esc'] / df_test['escolari']
def change(x):
x=x[-1:]
return x
df_=df.copy()
df_.ImageId_ClassId=df_.ImageId_ClassId.apply(change)
 df_[df_.EncodedPixels!=''].ImageId_ClassId.value_counts()<load_from_csv>
df_train['tabulet_x_qmobilephone'] = df_train['v18q1'] * df_train['qmobilephone']
df_test['tabulet_x_qmobilephone'] = df_test['v18q1'] * df_test['qmobilephone']
<categorify>
for col1 in ['epared1', 'epared2', 'epared3']:
for col2 in ['etecho1', 'etecho2', 'etecho3']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
for col1 in ['epared1', 'epared2', 'epared3']:
for col2 in ['eviv1', 'eviv2', 'eviv3']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
for col1 in ['etecho1', 'etecho2', 'etecho3']:
for col2 in ['eviv1', 'eviv2', 'eviv3']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
<load_from_csv>
for col1 in ['epared1', 'epared2', 'epared3']:
for col2 in ['etecho1', 'etecho2', 'etecho3']:
for col3 in ['eviv1', 'eviv2', 'eviv3']:
new_col_name = 'new_{}_x_{}_x_{}'.format(col1, col2, col3)
 df_train[new_col_name] = df_train[col1] * df_train[col2] * df_train[col3]
df_test[new_col_name] = df_test[col1] * df_test[col2] * df_train[col3]
df = pd.read_csv('.. /input/severstal-steel-defect-detection/sample_submission.csv')
 df['EncodedPixels']=['' for i in range(len(df)) ]
df_submit = pd.read_csv(".. /input/unet-sai/submission.csv")
 df_submit = df_submit.fillna('' )<feature_engineering>
for col1 in ['public', 'planpri', 'noelec', 'coopele']:
for col2 in ['energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
df_submit['ImageId_ClassId'] = [(str(df_submit['ImageId'][i])+'_'+str(df_submit['ClassId'][i])) for i in range(len(df_submit)) ]
df_submit.head()<save_to_csv>
for col1 in ['sanitario1', 'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6']:
for col2 in ['elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5', 'elimbasu6']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
if df.columns[0]=='ImageId_ClassId':
df.set_index('ImageId_ClassId', inplace=True)
 df_submit.set_index('ImageId_ClassId', inplace=True)
 for name, row in df_submit.iterrows() :
df.loc[name] = row
df.reset_index(inplace=True)
 df.to_csv('submission.csv', index=False)
 df.head()<import_modules>
for col1 in ['abastaguadentro', 'abastaguafuera', 'abastaguano']:
for col2 in ['sanitario1', 'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
%matplotlib inline 
<import_modules>
for col1 in ['area1', 'area2']:
for col2 in ['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
PadIfNeeded,
HorizontalFlip,
VerticalFlip,
CenterCrop,
Crop,
Compose,
Transpose,
RandomRotate90,
ElasticTransform,
GridDistortion,
OpticalDistortion,
RandomSizedCrop,
OneOf,
CLAHE,
RandomBrightnessContrast,
RandomGamma,
RandomContrast,
InvertImg,
RandomContrast
)<feature_engineering>
for col1 in ['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']:
for col2 in ['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
os.listdir('/kaggle/input/severstal-steel-defect-detection')
 train_df = pd.read_csv('/kaggle/input/severstal-steel-defect-detection/train.csv')
 train_df['Image_id']= train_df[train_df.columns[0]].apply(lambda x : x.split('_')[0])
 train_df['class_id']= train_df[train_df.columns[0]].apply(lambda x : x.split('_')[1])
 train_df['mask']= ~train_df[train_df.columns[1]].isna()
print(train_df.shape)
 train_df.head(5 )<groupby>
df_train['electronics'] = df_train['computer'] * df_train['mobilephone'] * df_train['television'] * df_train['v18q'] * df_train['refrig']
df_test['electronics'] = df_test['computer'] * df_test['mobilephone'] * df_test['television'] * df_test['v18q'] * df_test['refrig']
df_train['no_appliances'] = df_train['refrig'] + df_train['computer'] + df_train['television'] + df_train['mobilephone']
df_test['no_appliances'] = df_test['refrig'] + df_test['computer'] + df_test['television'] + df_test['mobilephone']
mask_count=train_df.groupby(by='Image_id' ).agg(np.sum ).reset_index()
print(mask_count.shape)
 mask_count.head(3 )<sort_values>
for col1 in ['paredblolad', 'paredzocalo', 'paredpreb', 'pareddes', 'paredmad', 'paredzinc', 'paredfibras', 'paredother']:
for col2 in ['pisomoscer', 'pisocemento', 'pisoother', 'pisonatur', 'pisonotiene', 'pisomadera']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
for col1 in ['pisomoscer', 'pisocemento', 'pisoother', 'pisonatur', 'pisonotiene', 'pisomadera']:
for col1 in ['techozinc', 'techoentrepiso', 'techocane', 'techootro']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
for col1 in ['paredblolad', 'paredzocalo', 'paredpreb', 'pareddes', 'paredmad', 'paredzinc', 'paredfibras', 'paredother']:
for col2 in ['techozinc', 'techoentrepiso', 'techocane', 'techootro']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
for col1 in ['paredblolad', 'paredzocalo', 'paredpreb', 'pareddes', 'paredmad', 'paredzinc', 'paredfibras', 'paredother']:
for col2 in ['pisomoscer', 'pisocemento', 'pisoother', 'pisonatur', 'pisonotiene', 'pisomadera']:
for col3 in ['techozinc', 'techoentrepiso', 'techocane', 'techootro']:
new_col_name = 'new_{}_x_{}_x_{}'.format(col1, col2, col3)
 df_train[new_col_name] = df_train[col1] * df_train[col2] * df_train[col3]
df_test[new_col_name] = df_test[col1] * df_test[col2] * df_train[col3]
mask_count.sort_values(by='mask',ascending=False,inplace=True)
 mask_count.head()<feature_engineering>
cols_with_only_one_value = []
for col in df_train.columns:
if col == 'Target':
continue
if df_train[col].value_counts().shape[0] == 1 or df_test[col].value_counts().shape[0] == 1:
print(col)
 cols_with_only_one_value.append(col )
sub_df = pd.read_csv('/kaggle/input/severstal-steel-defect-detection/sample_submission.csv')
 sub_df['Image_id']=sub_df[sub_df.columns[0]].apply(lambda x : x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['Image_id'].unique() ,columns=['Image_id'])
 print(test_imgs.shape)
 test_imgs.head()<define_variables>
df_train.drop(cols_with_only_one_value, axis=1, inplace=True)
 df_test.drop(cols_with_only_one_value, axis=1, inplace=True )
train_path = '/kaggle/input/severstal-steel-defect-detection/train_images/'
train_dir=os.listdir(train_path)
 train_dir[:3]
test_path = '/kaggle/input/severstal-steel-defect-detection/test_images/'
test_dir = os.listdir(test_path )<categorify>
cols_train = np.array(sorted([col for col in df_train.columns if col != 'Target']))
cols_test = np.array(sorted(df_test.columns))
def mask2rle(img):
pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(rle, input_shape):
width, height = input_shape[:2]
mask= np.zeros(width*height ).astype(np.uint8)
 array = np.asarray([int(x)for x in rle.split() ])
 starts = array[0::2]
lengths = array[1::2]
current_position = 0
for index, start in enumerate(starts):
mask[int(start):int(start+lengths[index])] = 1
current_position += lengths[index]
return mask.reshape(height, width ).T
def build_masks(rles, input_shape):
depth = len(rles)
 masks = np.zeros(( *input_shape, depth))
for i, rle in enumerate(rles):
if type(rle)is str:
masks[:, :, i] = rle2mask(rle, input_shape)
 return masks
def build_rles(masks):
width, height, depth = masks.shape
rles = [mask2rle(masks[:, :, i])
 for i in range(depth)]
return rles 
<choose_model_class>
agg_train = pd.DataFrame()
agg_test = pd.DataFrame()
for item in tqdm(family_size_features):
for i, function in enumerate(['mean','std','min','max','sum', 'count', max_min]):
group_train = df_train[item].groupby(df_train['idhogar'] ).agg(function)
 group_test = df_test[item].groupby(df_test['idhogar'] ).agg(function)
 if i == 6:
new_col = item + '_new_' + 'max_min'
else:
new_col = item + '_new_' + function
agg_train[new_col] = group_train
agg_test[new_col] = group_test
print('new aggregate train set has {} rows, and {} features'.format(agg_train.shape[0], agg_train.shape[1]))
print('new aggregate test set has {} rows, and {} features'.format(agg_test.shape[0], agg_test.shape[1]))
pa='/kaggle/input/efficientnet'
sys.path.append(os.path.abspath('/kaggle/input/efficientnet-models/efficientnet-master/'))
N_CLASSES=4
def ClsModel_2(n_classes=1, input_shape=(254,600,3)) :
base_model = EfficientNetB2(weights=None, include_top=False, input_shape=input_shape)
 x = AveragePooling2D(pool_size=(5,5), name='avg_pool' )(base_model.output)
 x = Flatten()(x)
 x = Dense(1600, activation='relu', name='dense_post_pool' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(n_classes, activation='sigmoid', name='predictions' )(x)
 model = Model(inputs=base_model.input, output=output)
 return model
model = ClsModel_2(n_classes=N_CLASSES)
 N_CLASSES=4
def ClsModel_3(n_classes=1, input_shape=(254,254,3)) :
base_model = EfficientNetB3(weights=None, include_top=False, input_shape=input_shape)
 x = AveragePooling2D(pool_size=(3,3), name='avg_pool' )(base_model.output)
 x = Flatten()(x)
 x = Dense(1024, activation='relu', name='dense_post_pool' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(n_classes, activation='sigmoid', name='predictions' )(x)
 model = Model(inputs=base_model.input, output=output)
 return model
model_class = ClsModel_3(n_classes=N_CLASSES)
 N_CLASSES=4
def ClsModel_1(n_classes=1, input_shape=(128,800,3)) :
base_model = EfficientNetB1(weights=None, include_top=False, input_shape=input_shape)
 x = AveragePooling2D(pool_size=(3,3), name='avg_pool' )(base_model.output)
 x = Flatten()(x)
 x = Dense(1024, activation='relu', name='dense_post_pool' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(n_classes, activation='sigmoid', name='predictions' )(x)
 model = Model(inputs=base_model.input, output=output)
 return model
model_class_b1 = ClsModel_1(n_classes=N_CLASSES)
<train_model>
aggr_list = ['rez_esc', 'dis', 'male', 'female',
'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',
'epared1', 'epared2', 'epared3', 'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3', 'refrig', 'television', 'mobilephone',
'area1', 'area2', 'v18q', 'edjef']
for item in tqdm(aggr_list):
for function in ['count', 'sum']:
group_train = df_train[item].groupby(df_train['idhogar'] ).agg(function)
 group_test = df_test[item].groupby(df_test['idhogar'] ).agg(function)
 new_col = item + '_new1_' + function
agg_train[new_col] = group_train
agg_test[new_col] = group_test
print('new aggregate train set has {} rows, and {} features'.format(agg_train.shape[0], agg_train.shape[1]))
print('new aggregate test set has {} rows, and {} features'.format(agg_test.shape[0], agg_test.shape[1]))
class DataGenerator1(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,val ='no',augmentaion =[1],
batch_size=1, dim=(254, 600), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.val=val
self.augmentaion=augmentaion
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[self.batch_size*index:self.batch_size*(index+1)]
ID = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(ID)
 if self.mode == 'fit':
y = self.__generate_y(ID)
 return X,y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,loc in enumerate(ID):
im_name = self.df['Image_id'].loc[loc]
img_path = f"{self.base_path}/{im_name}"
img=cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img=cv2.resize(img ,(600,254))
if self.augmentaion[0]==1 :
img_i[i,] = img
continue;
aug_no =0
aug = self.augmentaion[aug_no](p=1)
 img = aug(image=img)['image']
img_i[i,] = img
return img_i
def __generate_y(self, ID):
y = np.empty(( self.batch_size,self.n_classes))
for i,loc in enumerate(ID):
y[i,] = self.df['mask'].loc[loc]
return y
def __load_rgb(self, img_path):
return cv2.imread(img_path)
<train_model>
aggr_list = ['escolari', 'age', 'escolari_age', 'dependency', 'bedrooms', 'overcrowding', 'rooms', 'qmobilephone', 'v18q1']
for item in tqdm(aggr_list):
for function in ['mean','std','min','max','sum', 'count', max_min]:
group_train = df_train[item].groupby(df_train['idhogar'] ).agg(function)
 group_test = df_test[item].groupby(df_test['idhogar'] ).agg(function)
 if i == 6:
new_col = item + '_new2_' + 'max_min'
else:
new_col = item + '_new2_' + function
agg_train[new_col] = group_train
agg_test[new_col] = group_test
print('new aggregate train set has {} rows, and {} features'.format(agg_train.shape[0], agg_train.shape[1]))
print('new aggregate test set has {} rows, and {} features'.format(agg_test.shape[0], agg_test.shape[1]))
class DataGenerator2(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,val ='no',augmentaion =[1],
batch_size=1, dim=(254, 254), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.val=val
self.augmentaion=augmentaion
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[self.batch_size*index:self.batch_size*(index+1)]
ID = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(ID)
 if self.mode == 'fit':
y = self.__generate_y(ID)
 return X,y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,loc in enumerate(ID):
im_name = self.df['Image_id'].loc[loc]
img_path = f"{self.base_path}/{im_name}"
img=cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img=cv2.resize(img ,(254,254))
if self.augmentaion[0]==1 :
img_i[i,] = img
continue;
aug_no =0
aug = self.augmentaion[aug_no](p=1)
 img = aug(image=img)['image']
img_i[i,] = img
return img_i
def __generate_y(self, ID):
y = np.empty(( self.batch_size,self.n_classes))
for i,loc in enumerate(ID):
y[i,] = self.df['mask'].loc[loc]
return y
def __load_rgb(self, img_path):
return cv2.imread(img_path)
<train_model>
agg_test = agg_test.reset_index()
agg_train = agg_train.reset_index()
train_agg = pd.merge(df_train, agg_train, on='idhogar')
 test = pd.merge(df_test, agg_test, on='idhogar')
 train_agg.fillna(value=0, inplace=True)
 test.fillna(value=0, inplace=True)
 print('train shape:', train_agg.shape, 'test shape:', test.shape )
class DataGenerator3(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,val ='no',augmentaion =[1],
batch_size=1, dim=(128, 800), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.val=val
self.augmentaion=augmentaion
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[self.batch_size*index:self.batch_size*(index+1)]
ID = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(ID)
 if self.mode == 'fit':
y = self.__generate_y(ID)
 return X,y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,loc in enumerate(ID):
im_name = self.df['Image_id'].loc[loc]
img_path = f"{self.base_path}/{im_name}"
img=cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img=cv2.resize(img ,(800,128))
if self.augmentaion[0]==1:
img_i[i,] = img
continue;
aug_no = np.random.randint(0, len(self.augmentaion))
aug = self.augmentaion[aug_no](p=1)
 img = aug(image=img)['image']
img_i[i,] = img
return img_i
def __generate_y(self, ID):
y = np.empty(( self.batch_size,self.n_classes))
for i,loc in enumerate(ID):
y[i,] = self.df['mask'].loc[loc]
return y
def __load_rgb(self, img_path):
return cv2.imread(img_path)
<define_variables>
aggr_list = ['rez_esc', 'dis', 'male', 'female',
'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',
'epared1', 'epared2', 'epared3', 'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3', 'refrig', 'television', 'mobilephone',
'area1', 'area2', 'v18q', 'edjef']
for lugar in ['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']:
group_train = df_train[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).sum().reset_index()
group_train.columns = [lugar, 'idhogar'] + ['new3_{}_idhogar_{}'.format(lugar, col)for col in group_train][2:]
group_test = df_test[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).sum().reset_index()
group_test.columns = [lugar, 'idhogar'] + ['new3_{}_idhogar_{}'.format(lugar, col)for col in group_test][2:]
train_agg = pd.merge(train_agg, group_train, on=[lugar, 'idhogar'])
 test = pd.merge(test, group_test, on=[lugar, 'idhogar'])
 print('train shape:', train_agg.shape, 'test shape:', test.shape )
all_classifer_path=['/kaggle/input/efficent-b2-600/EfficientNetB3_efficenent_254_600_exp_4.h5',
'/kaggle/input/efficient977/EfficientNetB3_efficenent_exp_2.h5',
'/kaggle/input/efficient-b1-9793/EfficientNetB1_efficenent_128_800_exp_1.h5']
class_models =[model,model_class,model_class_b1]
pred_class_i=[]
pred=[]
DataGenerator=[DataGenerator1,DataGenerator2,DataGenerator3]
for i in range(2,3):
augmentaion=[1,HorizontalFlip,VerticalFlip]
for aug in augmentaion:
class_models[i].load_weights(all_classifer_path[i])
 test_generator = DataGenerator[i](
 test_imgs.index,
df=test_imgs,
shuffle=False,
mode='predict',
base_path='.. /input/severstal-steel-defect-detection/test_images',
target_df=test_imgs,
augmentaion=[aug],
batch_size=1,
n_classes=4 
)
 batch_pred_class = class_models[i].predict_generator(
 test_generator,
workers=1,
verbose=1,
use_multiprocessing=False 
)
 pred.append(batch_pred_class.round().astype(int)) 
<create_dataframe>
aggr_list = ['rez_esc', 'dis', 'male', 'female',
'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',
'epared1', 'epared2', 'epared3', 'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3', 'refrig', 'television', 'mobilephone',
'area1', 'area2', 'v18q', 'edjef']
for lugar in ['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']:
group_train = df_train[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).sum().reset_index()
group_train.columns = [lugar, 'idhogar'] + ['new4_{}_idhogar_{}'.format(lugar, col)for col in group_train][2:]
group_test = df_test[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).sum().reset_index()
group_test.columns = [lugar, 'idhogar'] + ['new4_{}_idhogar_{}'.format(lugar, col)for col in group_test][2:]
train_agg = pd.merge(train_agg, group_train, on=[lugar, 'idhogar'])
 test = pd.merge(test, group_test, on=[lugar, 'idhogar'])
 print('train shape:', train_agg.shape, 'test shape:', test.shape )
test=pred.copy()<count_values>
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
for function in tqdm(['mean','std','min','max','sum', 'count', max_min]):
for lugar in ['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']:
group_train = df_train[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).agg(function ).reset_index()
group_train.columns = [lugar, 'idhogar'] + ['new5_{}_idhogar_{}_{}'.format(lugar, col, function)for col in group_train][2:]
group_test = df_test[[lugar, 'idhogar'] + aggr_list].groupby([lugar, 'idhogar'] ).agg(function ).reset_index()
group_test.columns = [lugar, 'idhogar'] + ['new5_{}_idhogar_{}_{}'.format(lugar, col, function)for col in group_test][2:]
train_agg = pd.merge(train_agg, group_train, on=[lugar, 'idhogar'])
 test = pd.merge(test, group_test, on=[lugar, 'idhogar'])
 print('train shape:', train_agg.shape, 'test shape:', test.shape )
test_1=np.array(test)
 np.sum(test_1[0],axis=0 )<feature_engineering>
train = train_agg.query('parentesco1==1' )
test_1=np.array(test)
 test_1=test_1[0]+test_1[1]+test_1[2]
test_1[np.where(test_1<=1)]=0
test_1[np.where(test_1>1)]=1
np.sum(test_1,axis=0 )<data_type_conversions>
train['dependency'].replace(np.inf, 0, inplace=True)
 test['dependency'].replace(np.inf, 0, inplace=True )
np.sum(test_1.round().astype(int),axis=0 )<define_variables>
submission = test[['Id']]
train.drop(columns=['idhogar','Id', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 test.drop(columns=['idhogar','Id', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 correlation = train.corr()
correlation = correlation['Target'].sort_values(ascending=False )
aaa=[128/43/741/120]<count_values>
binary_cat_features = [col for col in train.columns if train[col].value_counts().shape[0] == 2]
object_features = ['edjefe', 'edjefa']
categorical_feats = binary_cat_features + object_features
np.sum(batch_pred_class )<data_type_conversions>
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True )
pred_class =test_1.round().astype(int )<groupby>
y = train['Target']
train.drop(columns=['Target'], inplace=True )
np.sum(pred_class,axis=0 )<train_model>
def print_execution_time(start):
end = time.time()
hours, rem = divmod(end-start, 3600)
 minutes, seconds = divmod(rem, 60)
 print('*'*20, "Execution ended in {:0>2}h {:0>2}m {:05.2f}s".format(int(hours),int(minutes),seconds), '*'*20 )
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,
batch_size=1, dim=(256, 1600), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,index in enumerate(ID):
im_name = self.df['Image_id'].loc[index]
img_path = f"{self.base_path}/{im_name}"
img =cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
 img_i[i,]=img
return img_i
def __generate_y(self, ID):
img_i = np.empty(( self.batch_size,*self.dim,self.n_classes),dtype=np.uint8)
 for i,index in enumerate(ID):
im_name = self.df['Image_id'].loc[index]
image_df = self.target_df[self.target_df['Image_id'] == im_name]
rles = image_df['EncodedPixels'].values
img =build_masks(rles, input_shape=self.dim)
 img_i[i,]=img
return img_i
def __check_input__(self,index):
X,y=self.__getitem__(index)
 indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
ID=list_IDs_batch[0]
X=X[0]
y=y[0]
im_name=self.df['Image_id'].loc[ID]
img= cv2.imread(train_path+im_name)
 image_df = self.target_df[self.target_df['Image_id'] == im_name]
rles = image_df['EncodedPixels'].values
masks = build_masks(rles, input_shape=self.dim)
 fig,axis = plt.subplots(nrows=1,ncols=1, figsize=(20, 20))
for i in range(1):
try:
img_i=mask_image(X[i].astype(np.uint8),y[i].astype(np.uint8))
axis[i].imshow(img_i)
 except IndexError:
print("IndexError : so skip")
 img_i=mask_image(X,y)
 axis.imshow(img_i,cmap='gray')
 break
except TypeError:
img_i=mask_image(X,y)
 print("i")
 axis.imshow(img_i,cmap='gray')
 print("TypeError(axis[i] is fails): so skip")
 break
img_i=mask_image(img.astype(np.uint8),masks.astype(np.uint8))
plt.figure(figsize=(10,10))
plt.imshow(img_i )<filter>
def extract_good_features_using_shap_LGB(params, SEED):
clf = lgb.LGBMClassifier(objective='multiclass',
random_state=1989,
max_depth=params['max_depth'],
learning_rate=params['learning_rate'],
silent=True,
metric='multi_logloss',
n_jobs=-1, n_estimators=10000,
class_weight='balanced',
colsample_bytree = params['colsample_bytree'],
min_split_gain= params['min_split_gain'],
bagging_freq = params['bagging_freq'],
min_child_weight=params['min_child_weight'],
num_leaves = params['num_leaves'],
subsample = params['subsample'],
reg_alpha= params['reg_alpha'],
reg_lambda= params['reg_lambda'],
num_class=len(np.unique(y)) ,
bagging_seed=SEED,
seed=SEED, 
)
 kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 feat_importance_df = pd.DataFrame()
for i,(train_index, test_index)in enumerate(kf.split(train, y)) :
print('='*30, '{} of {} folds'.format(i+1, kfold), '='*30)
 start = time.time()
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric=evaluate_macroF1_lgb, categorical_feature=categorical_feats,
early_stopping_rounds=500, verbose=500)
 shap_values = shap.TreeExplainer(clf.booster_ ).shap_values(X_train)
 fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = X_train.columns
fold_importance_df['shap_values'] = abs(np.array(shap_values)[:, :].mean(1 ).mean(0))
fold_importance_df['feat_imp'] = clf.feature_importances_
feat_importance_df = pd.concat([feat_importance_df, fold_importance_df])
 print_execution_time(start)
 feat_importance_df_shap = feat_importance_df.groupby('feature' ).mean().sort_values('shap_values', ascending=False ).reset_index()
return feat_importance_df_shap
non_missing_train_idx = mask_count[mask_count['mask'] > 0] 
<split>
total_shap_df = pd.DataFrame()
NUM_ITERATIONS = 50
for SEED in range(NUM_ITERATIONS):
print('
params = {'max_depth': np.random.choice([5, 6, 7, 8, 10, 12, -1]),
'learning_rate': np.random.rand() * 0.02,
'colsample_bytree': np.random.rand() *(1 - 0.5)+ 0.5,
'subsample': np.random.rand() *(1 - 0.5)+ 0.5,
'min_split_gain': np.random.rand() * 0.2,
'num_leaves': np.random.choice([32, 48, 64]),
'reg_alpha': np.random.rand() * 2,
'reg_lambda': np.random.rand() *2,
'bagging_freq': np.random.randint(4)+1,
'min_child_weight': np.random.randint(100)+ 20
}
temp_shap_df = extract_good_features_using_shap_LGB(params, SEED)
 total_shap_df = pd.concat([total_shap_df, temp_shap_df] )
BATCH_SIZE = 10
train_idx, val_idx = train_test_split(
 non_missing_train_idx.index,
random_state=25,
test_size=0.2 
)
 train_generator = DataGenerator(
 train_idx,
df=non_missing_train_idx,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4,
mode='fit' 
)
 val_generator = DataGenerator(
 val_idx,
df=non_missing_train_idx,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4 
)
 train_generator.__check_input__(0 )<compute_test_metric>
new_train = train[top_features].copy()
new_test = test[top_features].copy()
def dice_coef(y_true, y_pred, smooth= 1e-6):
y_pred = K.round(y_pred)
 intersection = K.sum(y_true * y_pred, axis=(1,2))
return K.mean(( 2.* intersection + smooth)/(K.sum(y_true, axis=(1,2)) + K.sum(y_pred, axis=(1,2)) + smooth))
def dice_loss(y_true, y_pred,smooth = 1e-9):
intersection = K.sum(y_true * y_pred, axis=(1,2))
score =(2.* intersection + smooth)/(K.sum(y_true, axis=(1,2)) + K.sum(y_pred, axis=(1,2)) + smooth)
 return K.mean(1-score)
 def bce_dice_loss(y_true, y_pred):
return dice_loss(y_true, y_pred )<compute_test_metric>
new_categorical_feats = [col for col in top_features if col in categorical_feats]
def dice_coef(y_true, y_pred, smooth= 1e-6):
y_pred = K.round(y_pred)
 intersection = K.sum(y_true * y_pred,axis=(1,2))
return K.mean(( 2.* intersection + smooth)/(K.sum(y_true, axis=(1,2)) + K.sum(y_pred, axis=(1,2)) + smooth))
opt=keras.optimizers.Adam(lr=1e-4)
<load_pretrained>
def LGB_OOF(params, categorical_feats, N_FOLDs, SEED=1989):
clf = lgb.LGBMClassifier(objective='multiclass',
random_state=1989,
max_depth=params['max_depth'],
learning_rate=params['learning_rate'],
silent=True,
metric='multi_logloss',
n_jobs=-1, n_estimators=10000,
class_weight='balanced',
colsample_bytree = params['colsample_bytree'],
min_split_gain= params['min_split_gain'],
bagging_freq = params['bagging_freq'],
min_child_weight=params['min_child_weight'],
num_leaves = params['num_leaves'],
subsample = params['subsample'],
reg_alpha= params['reg_alpha'],
reg_lambda= params['reg_lambda'],
num_class=len(np.unique(y)) ,
bagging_seed=SEED,
seed=SEED, 
)
 kfold = 10
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 feat_importance_df = pd.DataFrame()
predicts_result = []
for i,(train_index, test_index)in enumerate(kf.split(new_train, y)) :
print('='*30, '{} of {} folds'.format(i+1, kfold), '='*30)
 start = time.time()
X_train, X_val = new_train.iloc[train_index], new_train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric=evaluate_macroF1_lgb,categorical_feature=new_categorical_feats,
early_stopping_rounds=500, verbose=500)
 shap_values = shap.TreeExplainer(clf.booster_ ).shap_values(X_train)
 fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = X_train.columns
fold_importance_df['shap_values'] = abs(np.array(shap_values)[:, :].mean(1 ).mean(0))
fold_importance_df['feat_imp'] = clf.feature_importances_
feat_importance_df = pd.concat([feat_importance_df, fold_importance_df])
 predicts_result.append(clf.predict(new_test))
print_execution_time(start)
 return predicts_result, feat_importance_df
model_seg = load_model('/kaggle/input/seg-model/seg_model.h5')
 model_seg.load_weights('/kaggle/input/16000-model/1600.h5' )<groupby>
params = {'max_depth': 6,
'learning_rate': 0.002,
'colsample_bytree': 0.8,
'subsample': 0.8,
'min_split_gain': 0.02,
'num_leaves': 48,
'reg_alpha': 0.04,
'reg_lambda': 0.073,
'bagging_freq': 2,
'min_child_weight': 40
}
N_Folds = 20
SEED = 1989
predicts_result, feat_importance_df = LGB_OOF(params, new_categorical_feats, N_Folds, SEED=1989 )
np.sum(pred_class,axis=0 )<feature_engineering>
submission['Target'] = np.array(predicts_result ).mean(axis=0 ).round().astype(int)
 submission.to_csv('submission_with_new_feature_set.csv', index = False )
sub_df = pd.read_csv('.. /input/severstal-steel-defect-detection/sample_submission.csv')
 sub_df['Image_id'] = sub_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['Image_id'].unique() , columns=['Image_id'])
 test_imgs=test_imgs
filtered_mask = sub_df['Image_id'].isin(test_imgs["Image_id"].values)
 filtered_sub_df = sub_df[filtered_mask].copy()
null_sub_df = sub_df[~filtered_mask].copy()
null_sub_df['EncodedPixels'] = null_sub_df['EncodedPixels'].apply(
 lambda x: ' ')
 filtered_sub_df.reset_index(drop=True, inplace=True)
 test_imgs.reset_index(drop=True, inplace=True)
 print(filtered_sub_df.shape)
 print(null_sub_df.shape)
 filtered_sub_df.head() 
<categorify>
optimized_param = None
lowest_cv = 1000
total_iteration = 100
for i in range(total_iteration):
print('-'*20, 'For {} of {} iterations'.format(i+1, total_iteration), '-'*20)
 learning_rate = np.random.rand() * 0.02
n_folds = 3
num_class = len(np.unique(y))
params = {}
params['application'] = 'multiclass'
params['metric'] = 'multi_logloss'
params['num_class'] = num_class
params['class_weight'] = 'balanced'
params['num_leaves'] = np.random.randint(24, 48)
 params['max_depth'] = np.random.randint(5, 8)
 params['min_child_weight'] = np.random.randint(5, 50)
 params['min_split_gain'] = np.random.rand() * 0.09
params['colsample_bytree'] = np.random.rand() *(0.9 - 0.1)+ 0.1
params['subsample'] = np.random.rand() *(1 - 0.8)+ 0.8
params['bagging_freq'] = np.random.randint(1, 5)
 params['bagging_seed'] = np.random.randint(1, 5)
 params['reg_alpha'] = np.random.rand() * 2
params['reg_lambda'] = np.random.rand() * 2
params['learning_rate'] = np.random.rand() * 0.02
params['seed'] =1989
d_train = lgb.Dataset(data=new_train, label=y.values-1, categorical_feature=new_categorical_feats, free_raw_data=False)
 cv_results = lgb.cv(params=params, train_set=d_train, num_boost_round=10000, categorical_feature=new_categorical_feats,
nfold=n_folds, stratified=True, shuffle=True, early_stopping_rounds=1, verbose_eval=1000)
 min_cv_results = min(cv_results['multi_logloss-mean'])
 if min_cv_results < lowest_cv:
lowest_cv = min_cv_results
optimized_param = params
all_class=pred_class.flatten()
all_class
filtered_sub_df['EncodedPixels']=all_class<define_variables>
N_Folds = 20
SEED = 1989
predicts_result, feat_importance_df = LGB_OOF(optimized_param, new_categorical_feats, N_Folds, SEED=1989 )
<train_model><EOS> 
submission['Target'] = np.array(predicts_result ).mean(axis=0 ).round().astype(int)
 submission.to_csv('submission_shap_randomized_search.csv', index = False )