Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<prepare_x_and_y>
%matplotlib inline
datagen = ImageDataGenerator(preprocessing_function=preprocess_input, rescale=1./255., horizontal_flip=True,width_shift_range = 0.2,height_shift_range = 0.2,validation_split=0.15)
 train_generator=datagen.flow_from_dataframe(dataframe=df,directory=".. /input/train/",x_col="id",y_col="breed",has_ext=False,subset="training",batch_size=16,seed=1997,shuffle=True,class_mode="categorical",target_size=(image_size, image_size))
valid_generator=datagen.flow_from_dataframe(dataframe=df,directory=".. /input/train/",x_col="id",y_col="breed",has_ext=False,subset="validation",batch_size=1,seed=1997,shuffle=True,class_mode="categorical",target_size=(image_size, image_size))
test_datagen=ImageDataGenerator(preprocessing_function=preprocess_input,rescale=1./255.)
test_generator=test_datagen.flow_from_dataframe(dataframe=test_df,directory=".. /input/test/",x_col="id",y_col=None,has_ext=False,batch_size=1,seed=42,shuffle=False,class_mode=None,target_size=(image_size, image_size))<train_model>
test = pd.read_csv(".. /input/test.csv")
 poverty_train = pd.read_csv(".. /input/train.csv")
 display(poverty_train.head() , poverty_train.shape )
model.fit_generator(generator=train_generator,steps_per_epoch=train_generator.n,validation_data=valid_generator,validation_steps=train_generator.n,epochs=1 )<predict_on_test>
print("Top Columns having missing values in the train set.")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()
pred=model.predict_generator(test_generator,verbose=1 )<save_to_csv>
print("Top Columns having missing values in the test set.")
 missmap = test.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(test)
 missmap.head()
labels =(train_generator.class_indices)
 labels = list(labels.keys())
 df = pd.DataFrame(data=pred,
columns=labels)
 columns = list(df)
 columns.sort()
df = df.reindex(columns=columns)
 filenames = test_df["id"]
df["id"] = filenames
cols = df.columns.tolist()
cols = cols[-1:] + cols[:-1]
df = df[cols]
df.head(5)
 df.to_csv("submission_InceptionResnet50.csv",index=False )<set_options>
poverty_train[poverty_train.meaneduc.isnull() ]
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<load_from_csv>
value1 = poverty_train["meaneduc"].mean()
value2 = value1*value1
display(value1, value2)
labels = pd.read_csv(".. /input/labels.csv" )<define_variables>
poverty_train["meaneduc"].fillna(value1, inplace = True)
 poverty_train["SQBmeaned"].fillna(value2, inplace = True)
 print("Top Columns having missing values")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()
PATH = ".. /input/"
TMP_PATH = "/tmp/tmp"
MODEL_PATH = "/tmp/model/"<set_options>
display(poverty_train.loc[(poverty_train["tipovivi1" ] == 1, "v2a1")].isna().sum() ,
len(poverty_train.loc[(poverty_train["tipovivi1" ] == 1, "v2a1")]))
arch=resnet101
sz=224
bs=48<choose_model_class>
poverty_train.loc[(poverty_train["tipovivi1" ] == 1, "v2a1")] = 0
print("Top Columns having missing values")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()
tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1 )<load_from_csv>
display(poverty_train.loc[(poverty_train["tipovivi2" ] == 1, "v2a1")].isna().sum() ,
len(poverty_train.loc[(poverty_train["tipovivi2" ] == 1, "v2a1")]))
data = ImageClassifierData.from_csv(path=PATH, folder="train", csv_fname=f"{PATH}labels.csv", tfms=tfms, suffix=".jpg", test_name="test",bs=bs,num_workers=4 )<load_pretrained>
display(poverty_train.loc[(poverty_train["tipovivi3" ] == 1, "v2a1")].isna().sum() ,
len(poverty_train.loc[(poverty_train["tipovivi3" ] == 1, "v2a1")]))
learn = ConvLearner.pretrained(arch, data, precompute=True, tmp_name=TMP_PATH, models_name=MODEL_PATH, ps=0.4 )<find_best_params>
display(poverty_train.loc[(poverty_train["tipovivi4" ] == 1, "v2a1")].isna().sum() ,
len(poverty_train.loc[(poverty_train["tipovivi4" ] == 1, "v2a1")]))
lrf = learn.lr_find()
learn.sched.plot() 
<load_from_csv>
display(poverty_train.loc[(poverty_train["tipovivi5" ] == 1, "v2a1")].isna().sum() ,
len(poverty_train.loc[(poverty_train["tipovivi5" ] == 1, "v2a1")]))
def get_data(sz,bs):
tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom = 1.1)
 data = ImageClassifierData.from_csv(PATH, 'train', f'{PATH}labels.csv', test_name='test', suffix='.jpg',tfms=tfms, bs=bs, num_workers=4)
 return data if sz > 300 else data.resize(sz, '/tmp' )<train_model>
poverty_train["v2a1"].fillna(0, inplace = True)
 print("Top Columns having missing values(%)")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()
learn.fit(1e-1, 5)
<train_model>
display(poverty_train.loc[(poverty_train["v18q" ] == 0, "v18q1")].isna().sum() ,
len(poverty_train[poverty_train["v18q"] == 0]))
learn.precompute=False
learn.fit(1e-1, 5, cycle_len=1 )<set_options>
poverty_train.loc[(poverty_train["v18q" ] == 0, "v18q1")] = 0
print("Top Columns having missing values(%)")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()
learn.set_data(get_data(299,bs))<train_model>
poverty_train['rez_esc'] = poverty_train.apply(
 lambda row: 0 if(row['instlevel1'] == 1)else row['rez_esc'],
axis=1 
)
 poverty_train['rez_esc'] = poverty_train.apply(
 lambda row: poverty_train["rez_esc"].median() if np.isnan(row["rez_esc"])else row['rez_esc'],
axis=1 
)
 print("Top Columns having missing values(%)")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()
learn.fit(1e-1, 3, cycle_len=1 )<train_model>
poverty_train["dependency"].replace('yes', 1, inplace = True)
 poverty_train["dependency"].replace('no', 0, inplace = True)
 poverty_train["edjefa"].replace('yes', 1, inplace = True)
 poverty_train["edjefa"].replace('no', 0, inplace = True)
 poverty_train["edjefe"].replace('yes', 1, inplace = True)
 poverty_train["edjefe"].replace('no', 0, inplace = True)
 display(poverty_train["dependency"].value_counts() ,
poverty_train["edjefa"].value_counts() ,
poverty_train["edjefe"].value_counts() )
learn.fit(1e-1,3,cycle_len=1,cycle_mult=2 )<train_model>
poverty_train["edjefe"] = poverty_train["edjefe"].astype(float)
 poverty_train["edjefa"] = poverty_train["edjefa"].astype(float)
 poverty_train["dependency"] = poverty_train["dependency"].astype(float )
learn.fit(1e-1, 1, cycle_len=2 )<compute_test_metric>
d={}
weird=[]
for row in poverty_train.iterrows() :
idhogar=row[1]['idhogar']
target=row[1]['Target']
if idhogar in d:
if d[idhogar]!=target:
weird.append(idhogar)
 else:
d[idhogar]=target
len(set(weird))
log_preds, y = learn.TTA()
probs = np.mean(np.exp(log_preds), 0)
 accuracy_np(probs, y), log_loss(y, probs )<save_to_csv>
for i in set(weird):
hhold=poverty_train[poverty_train['idhogar']==i][['idhogar', 'parentesco1', 'Target']]
target=hhold[hhold['parentesco1']==1]['Target'].tolist() [0]
for row in hhold.iterrows() :
idx=row[0]
if row[1]['parentesco1']!=1:
poverty_train.at[idx, 'Target']=target
log_preds_test, y_test = learn.TTA(is_test=True)
 probs_test = np.mean(np.exp(log_preds_test), 0)
 df = pd.DataFrame(probs_test)
 df.columns = data.classes
df.insert(0, "id", [e[5:-4] for e in data.test_ds.fnames])
 df.to_csv("submission.csv", index=False )<set_options>
poverty_train[poverty_train['idhogar']==weird[1]][['idhogar','parentesco1', 'Target']]
s_time = time.time()
warnings.filterwarnings('ignore')
<categorify>
very_poor = poverty_train[poverty_train["Target"] == 1]
poor = poverty_train[poverty_train["Target"] == 2]
vulnerable = poverty_train[poverty_train["Target"] == 3]
safe = poverty_train[poverty_train["Target"] == 4]
def post_process(mask, min_size):
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predict = np.zeros(( 256, 1600), np.float32)
 for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predict[p] = 1
return predict
def probability_mask_to_probability_label(probability):
probability = F.adaptive_max_pool2d(probability,1 ).squeeze(-1 ).squeeze(-1)
 return probability
def run_length_encode(mask):
m = mask.T.flatten()
if m.sum() ==0:
rle=''
else:
m = np.concatenate([[0], m, [0]])
 run = np.where(m[1:] != m[:-1])[0] + 1
run[1::2] -= run[::2]
rle = ' '.join(str(r)for r in run)
 return rle<set_options>
def mean_confidence_interval(data, confidence=0.95):
a = 1.0*np.array(data)
 n = len(a)
 m, se = np.mean(a), scipy.stats.sem(a)
 h = se * sp.stats.t._ppf(( 1+confidence)/2., n-1)
 return m, m-h, m+h
class SteelDataset(Dataset):
def __init__(self, defect_root,
resize_long=None, img_normal=True, hor_flip=False,
crop_size=None, to_torch=True, random_hue=False):
self.img_name_list = glob.glob(defect_root + '/*.jpg')
 self.defect_root = defect_root
self.resize_long = resize_long
self.crop_size = crop_size
self.img_normal = img_normal
self.hor_flip = hor_flip
self.to_torch = to_torch
self.random_hue = random_hue
def __len__(self):
return len(self.img_name_list)
 def __getitem__(self, idx):
name = self.img_name_list[idx]
img = Image.open(name)
 if self.resize_long:
target_long = random.randint(self.resize_long[0], self.resize_long[1])
 img = transforms.Resize(( 256, target_long))(img)
 if self.hor_flip:
img = transforms.RandomHorizontalFlip()(img)
 if self.crop_size:
img = transforms.RandomCrop(( self.crop_size[0], self.crop_size[1]), padding=0, pad_if_needed=True )(img)
 if self.random_hue:
img = transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.25 )(img)
 if self.to_torch:
img = transforms.ToTensor()(img)
 if self.img_normal:
img = transforms.Normalize(mean=(0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225))(img)
 return {'name': name, 'img': img}
class SteelDatasetMSF(Dataset):
def __init__(self, defect_root,
resize_long=None, img_normal=True, hor_flip=False,
crop_size=None, to_torch=True, random_hue=False, scales=[1.]):
self.img_name_list = glob.glob(defect_root + '/*.jpg')
 self.defect_root = defect_root
self.scales = scales
self.resize_long = resize_long
self.crop_size = crop_size
self.img_normal = img_normal
self.hor_flip = hor_flip
self.to_torch = to_torch
self.random_hue = random_hue
def __len__(self):
return len(self.img_name_list)
 def __getitem__(self, idx):
name = self.img_name_list[idx]
img = Image.open(name)
 ori_w, ori_h = img.size
ms_img_list = []
for s in self.scales:
if s == 1:
s_img = img
else:
w = int(s*ori_w)
 s_img = transforms.Resize(( ori_h, w))(img)
 if self.random_hue:
s_img = transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.25 )(s_img)
 if self.to_torch:
s_img = transforms.ToTensor()(s_img)
 if self.img_normal:
s_img = transforms.Normalize(mean=(0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225))(s_img)
 ms_img_list.append(s_img)
 if len(self.scales)== 1:
ms_img_list = ms_img_list[0]
return {'name': name, 'img': ms_img_list}<categorify>
display(mean_confidence_interval(very_poor["age"], confidence = 0.95),
mean_confidence_interval(safe["age"], confidence = 0.95))
BatchNorm2d = nn.BatchNorm2d
BN_MOMENTUM = 0.01
def conv3x3(in_planes, out_planes, stride=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
 class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self ).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=False)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = out + residual
out = self.relu(out)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
 self.bn1 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
padding=1, bias=False)
 self.bn2 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1,
bias=False)
 self.bn3 = BatchNorm2d(planes * self.expansion,
momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=False)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = out + residual
out = self.relu(out)
 return out
class HighResolutionModule(nn.Module):
def __init__(self, num_branches, blocks, num_blocks, num_inchannels,
num_channels, fuse_method, multi_scale_output=True):
super(HighResolutionModule, self ).__init__()
self._check_branches(
 num_branches, blocks, num_blocks, num_inchannels, num_channels)
 self.num_inchannels = num_inchannels
self.fuse_method = fuse_method
self.num_branches = num_branches
self.multi_scale_output = multi_scale_output
self.branches = self._make_branches(
 num_branches, blocks, num_blocks, num_channels)
 self.fuse_layers = self._make_fuse_layers()
self.relu = nn.ReLU(inplace=False)
 def _check_branches(self, num_branches, blocks, num_blocks,
num_inchannels, num_channels):
if num_branches != len(num_blocks):
error_msg = 'NUM_BRANCHES({})<> NUM_BLOCKS({})'.format(
 num_branches, len(num_blocks))
logger.error(error_msg)
 raise ValueError(error_msg)
 if num_branches != len(num_channels):
error_msg = 'NUM_BRANCHES({})<> NUM_CHANNELS({})'.format(
 num_branches, len(num_channels))
logger.error(error_msg)
 raise ValueError(error_msg)
 if num_branches != len(num_inchannels):
error_msg = 'NUM_BRANCHES({})<> NUM_INCHANNELS({})'.format(
 num_branches, len(num_inchannels))
logger.error(error_msg)
 raise ValueError(error_msg)
 def _make_one_branch(self, branch_index, block, num_blocks, num_channels,
stride=1):
downsample = None
if stride != 1 or \
self.num_inchannels[branch_index] != num_channels[branch_index] * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.num_inchannels[branch_index],
num_channels[branch_index] * block.expansion,
kernel_size=1, stride=stride, bias=False),
BatchNorm2d(num_channels[branch_index] * block.expansion,
momentum=BN_MOMENTUM), 
)
 layers = []
layers.append(block(self.num_inchannels[branch_index],
num_channels[branch_index], stride, downsample))
self.num_inchannels[branch_index] = \
num_channels[branch_index] * block.expansion
for i in range(1, num_blocks[branch_index]):
layers.append(block(self.num_inchannels[branch_index],
num_channels[branch_index]))
return nn.Sequential(*layers)
 def _make_branches(self, num_branches, block, num_blocks, num_channels):
branches = []
for i in range(num_branches):
branches.append(
 self._make_one_branch(i, block, num_blocks, num_channels))
return nn.ModuleList(branches)
 def _make_fuse_layers(self):
if self.num_branches == 1:
return None
num_branches = self.num_branches
num_inchannels = self.num_inchannels
fuse_layers = []
for i in range(num_branches if self.multi_scale_output else 1):
fuse_layer = []
for j in range(num_branches):
if j > i:
fuse_layer.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_inchannels[i],
1,
1,
0,
bias=False),
BatchNorm2d(num_inchannels[i], momentum=BN_MOMENTUM)))
 elif j == i:
fuse_layer.append(None)
 else:
conv3x3s = []
for k in range(i - j):
if k == i - j - 1:
num_outchannels_conv3x3 = num_inchannels[i]
conv3x3s.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_outchannels_conv3x3,
3, 2, 1, bias=False),
BatchNorm2d(num_outchannels_conv3x3,
momentum=BN_MOMENTUM)))
 else:
num_outchannels_conv3x3 = num_inchannels[j]
conv3x3s.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_outchannels_conv3x3,
3, 2, 1, bias=False),
BatchNorm2d(num_outchannels_conv3x3,
momentum=BN_MOMENTUM),
nn.ReLU(inplace=False)))
 fuse_layer.append(nn.Sequential(*conv3x3s))
fuse_layers.append(nn.ModuleList(fuse_layer))
return nn.ModuleList(fuse_layers)
 def get_num_inchannels(self):
return self.num_inchannels
def forward(self, x):
if self.num_branches == 1:
return [self.branches[0](x[0])]
for i in range(self.num_branches):
x[i] = self.branches[i](x[i])
 x_fuse = []
for i in range(len(self.fuse_layers)) :
y = x[0] if i == 0 else self.fuse_layers[i][0](x[0])
 for j in range(1, self.num_branches):
if i == j:
y = y + x[j]
elif j > i:
width_output = x[i].shape[-1]
height_output = x[i].shape[-2]
y = y + F.interpolate(
 self.fuse_layers[i][j](x[j]),
size=[height_output, width_output],
mode='bilinear')
 else:
y = y + self.fuse_layers[i][j](x[j])
 x_fuse.append(self.relu(y))
return x_fuse
blocks_dict = {
'BASIC': BasicBlock,
'BOTTLENECK': Bottleneck
}
class HighResolutionNet(nn.Module):
def __init__(self, config, **kwargs):
extra = config['MODEL']['EXTRA']
super(HighResolutionNet, self ).__init__()
self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1,
bias=False)
 self.bn1 = BatchNorm2d(64, momentum=BN_MOMENTUM)
 self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=2, padding=1,
bias=False)
 self.bn2 = BatchNorm2d(64, momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=False)
 self.stage1_cfg = extra['STAGE1']
num_channels = self.stage1_cfg['NUM_CHANNELS'][0]
block = blocks_dict[self.stage1_cfg['BLOCK']]
num_blocks = self.stage1_cfg['NUM_BLOCKS'][0]
self.layer1 = self._make_layer(block, 64, num_channels, num_blocks)
 stage1_out_channel = block.expansion * num_channels
self.stage2_cfg = extra['STAGE2']
num_channels = self.stage2_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage2_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition1 = self._make_transition_layer(
 [stage1_out_channel], num_channels)
 self.stage2, pre_stage_channels = self._make_stage(
 self.stage2_cfg, num_channels)
 self.stage3_cfg = extra['STAGE3']
num_channels = self.stage3_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage3_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition2 = self._make_transition_layer(
 pre_stage_channels, num_channels)
 self.stage3, pre_stage_channels = self._make_stage(
 self.stage3_cfg, num_channels)
 self.stage4_cfg = extra['STAGE4']
num_channels = self.stage4_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage4_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition3 = self._make_transition_layer(
 pre_stage_channels, num_channels)
 self.stage4, pre_stage_channels = self._make_stage(
 self.stage4_cfg, num_channels, multi_scale_output=True)
 last_inp_channels = np.int(np.sum(pre_stage_channels))
self.last_layer = nn.Sequential(
 nn.Conv2d(
 in_channels=last_inp_channels,
out_channels=last_inp_channels,
kernel_size=1,
stride=1,
padding=0),
BatchNorm2d(last_inp_channels, momentum=BN_MOMENTUM),
nn.ReLU(inplace=False),
nn.Conv2d(
 in_channels=last_inp_channels,
out_channels=config['MODEL']['NUM_CLASSES'],
kernel_size=extra['FINAL_CONV_KERNEL'],
stride=1,
padding=1 if extra['FINAL_CONV_KERNEL'] == 3 else 0)
)
 self.backbone = nn.ModuleList([self.conv1, self.bn1, self.conv2, self.bn2, self.layer1, self.transition1,
self.stage2, self.transition2, self.stage3, self.transition3, self.stage4])
 self.fuse = nn.ModuleList([self.last_layer])
 def _make_transition_layer(
 self, num_channels_pre_layer, num_channels_cur_layer):
num_branches_cur = len(num_channels_cur_layer)
 num_branches_pre = len(num_channels_pre_layer)
 transition_layers = []
for i in range(num_branches_cur):
if i < num_branches_pre:
if num_channels_cur_layer[i] != num_channels_pre_layer[i]:
transition_layers.append(nn.Sequential(
 nn.Conv2d(num_channels_pre_layer[i],
num_channels_cur_layer[i],
3,
1,
1,
bias=False),
BatchNorm2d(
 num_channels_cur_layer[i], momentum=BN_MOMENTUM),
nn.ReLU(inplace=False)))
 else:
transition_layers.append(None)
 else:
conv3x3s = []
for j in range(i + 1 - num_branches_pre):
inchannels = num_channels_pre_layer[-1]
outchannels = num_channels_cur_layer[i] \
if j == i - num_branches_pre else inchannels
conv3x3s.append(nn.Sequential(
 nn.Conv2d(
 inchannels, outchannels, 3, 2, 1, bias=False),
BatchNorm2d(outchannels, momentum=BN_MOMENTUM),
nn.ReLU(inplace=False)))
 transition_layers.append(nn.Sequential(*conv3x3s))
return nn.ModuleList(transition_layers)
 def _make_layer(self, block, inplanes, planes, blocks, stride=1):
downsample = None
if stride != 1 or inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False),
BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM), 
)
 layers = []
layers.append(block(inplanes, planes, stride, downsample))
inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(inplanes, planes))
return nn.Sequential(*layers)
 def _make_stage(self, layer_config, num_inchannels,
multi_scale_output=True):
num_modules = layer_config['NUM_MODULES']
num_branches = layer_config['NUM_BRANCHES']
num_blocks = layer_config['NUM_BLOCKS']
num_channels = layer_config['NUM_CHANNELS']
block = blocks_dict[layer_config['BLOCK']]
fuse_method = layer_config['FUSE_METHOD']
modules = []
for i in range(num_modules):
if not multi_scale_output and i == num_modules - 1:
reset_multi_scale_output = False
else:
reset_multi_scale_output = True
modules.append(
 HighResolutionModule(num_branches,
block,
num_blocks,
num_inchannels,
num_channels,
fuse_method,
reset_multi_scale_output)
)
 num_inchannels = modules[-1].get_num_inchannels()
return nn.Sequential(*modules), num_inchannels
def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.conv2(x)
 x = self.bn2(x)
 x = self.relu(x)
 x = self.layer1(x)
 x_list = []
for i in range(self.stage2_cfg['NUM_BRANCHES']):
if self.transition1[i] is not None:
x_list.append(self.transition1[i](x))
else:
x_list.append(x)
 y_list = self.stage2(x_list)
 x_list = []
for i in range(self.stage3_cfg['NUM_BRANCHES']):
if self.transition2[i] is not None:
if i < self.stage2_cfg['NUM_BRANCHES']:
x_list.append(self.transition2[i](y_list[i]))
else:
x_list.append(self.transition2[i](y_list[-1]))
else:
x_list.append(y_list[i])
 y_list = self.stage3(x_list)
 x_list = []
for i in range(self.stage4_cfg['NUM_BRANCHES']):
if self.transition3[i] is not None:
if i < self.stage3_cfg['NUM_BRANCHES']:
x_list.append(self.transition3[i](y_list[i]))
else:
x_list.append(self.transition3[i](y_list[-1]))
else:
x_list.append(y_list[i])
 x = self.stage4(x_list)
 x0_h, x0_w = x[0].size(2), x[0].size(3)
 x1 = F.upsample(x[1], size=(x0_h, x0_w), mode='bilinear')
 x2 = F.upsample(x[2], size=(x0_h, x0_w), mode='bilinear')
 x3 = F.upsample(x[3], size=(x0_h, x0_w), mode='bilinear')
 x = torch.cat([x[0], x1, x2, x3], 1)
 x = self.last_layer(x)
 return x
def trainable_parameters(self):
return list(self.backbone.parameters()), list(self.fuse.parameters())
 def init_weights(self, pretrained='',):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.normal_(m.weight, std=0.001)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 if os.path.isfile(pretrained):
pretrained_dict = torch.load(pretrained)
 model_dict = self.state_dict()
pretrained_dict = {k: v for k, v in pretrained_dict.items()
if k in model_dict.keys() }
print('length of pretrained_dict:', len(pretrained_dict.items()))
model_dict.update(pretrained_dict)
 self.load_state_dict(model_dict)
 class CAM(HighResolutionNet):
def __init__(self, cfg, **kwargs):
super(CAM, self ).__init__(cfg, **kwargs)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.conv2(x)
 x = self.bn2(x)
 x = self.relu(x)
 x = self.layer1(x)
 x_list = []
for i in range(self.stage2_cfg['NUM_BRANCHES']):
if self.transition1[i] is not None:
x_list.append(self.transition1[i](x))
else:
x_list.append(x)
 y_list = self.stage2(x_list)
 x_list = []
for i in range(self.stage3_cfg['NUM_BRANCHES']):
if self.transition2[i] is not None:
if i < self.stage2_cfg['NUM_BRANCHES']:
x_list.append(self.transition2[i](y_list[i]))
else:
x_list.append(self.transition2[i](y_list[-1]))
else:
x_list.append(y_list[i])
 y_list = self.stage3(x_list)
 x_list = []
for i in range(self.stage4_cfg['NUM_BRANCHES']):
if self.transition3[i] is not None:
if i < self.stage3_cfg['NUM_BRANCHES']:
x_list.append(self.transition3[i](y_list[i]))
else:
x_list.append(self.transition3[i](y_list[-1]))
else:
x_list.append(y_list[i])
 x = self.stage4(x_list)
 x0_h, x0_w = x[0].size(2), x[0].size(3)
 x1 = F.upsample(x[1], size=(x0_h, x0_w), mode='bilinear')
 x2 = F.upsample(x[2], size=(x0_h, x0_w), mode='bilinear')
 x3 = F.upsample(x[3], size=(x0_h, x0_w), mode='bilinear')
 x = torch.cat([x[0], x1, x2, x3], 1)
 x = self.last_layer(x)
 x = F.relu(x)
 x = x[0] + x[1].flip(-1)
 return x
def get_hr_seg_model(cfg, **kwargs):
cfg['MODEL']['NUM_CLASSES'] = 5
model = HighResolutionNet(cfg, **kwargs)
 return model<define_search_model>
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop(['Id'], axis=1, inplace=True)
 df.drop(['hhsize', 'female', 'area2'], axis=1, inplace=True)
 return df
def convert_OHE2LE(df):
print_check = True
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
if print_check:
sum_ohe = df[cols_s_].sum(axis=1 ).unique()
if sum_ohe.shape[0]>1:
print(s_)
 print(df[cols_s_].sum(axis=1 ).value_counts())
 tmp_cat = df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df
ALIGN_CORNERS = True
BN_MOMENTUM = 0.1
relu_inplace = True
BatchNorm2d_class = BatchNorm2d = torch.nn.BatchNorm2d
class ModuleHelper:
@staticmethod
def BNReLU(num_features, bn_type=None, **kwargs):
return nn.Sequential(
 BatchNorm2d(num_features, **kwargs),
nn.ReLU() 
)
 @staticmethod
def BatchNorm2d(*args, **kwargs):
return BatchNorm2d
def conv3x3(in_planes, out_planes, stride=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
 class SpatialGather_Module(nn.Module):

 def __init__(self, cls_num=0, scale=1):
super(SpatialGather_Module, self ).__init__()
self.cls_num = cls_num
self.scale = scale
def forward(self, feats, probs):
batch_size, c, h, w = probs.size(0), probs.size(1), probs.size(2), probs.size(3)
 probs = probs.view(batch_size, c, -1)
 feats = feats.view(batch_size, feats.size(1), -1)
 feats = feats.permute(0, 2, 1)
 probs = F.softmax(self.scale * probs, dim=2)
 ocr_context = torch.matmul(probs, feats)\ 
.permute(0, 2, 1 ).unsqueeze(3)
 return ocr_context
class _ObjectAttentionBlock(nn.Module):

 def __init__(self,
in_channels,
key_channels,
scale=1,
bn_type=None):
super(_ObjectAttentionBlock, self ).__init__()
self.scale = scale
self.in_channels = in_channels
self.key_channels = key_channels
self.pool = nn.MaxPool2d(kernel_size=(scale, scale))
self.f_pixel = nn.Sequential(
 nn.Conv2d(in_channels=self.in_channels, out_channels=self.key_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type),
nn.Conv2d(in_channels=self.key_channels, out_channels=self.key_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), 
)
 self.f_object = nn.Sequential(
 nn.Conv2d(in_channels=self.in_channels, out_channels=self.key_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type),
nn.Conv2d(in_channels=self.key_channels, out_channels=self.key_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), 
)
 self.f_down = nn.Sequential(
 nn.Conv2d(in_channels=self.in_channels, out_channels=self.key_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), 
)
 self.f_up = nn.Sequential(
 nn.Conv2d(in_channels=self.key_channels, out_channels=self.in_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.in_channels, bn_type=bn_type), 
)
 def forward(self, x, proxy):
batch_size, h, w = x.size(0), x.size(2), x.size(3)
 if self.scale > 1:
x = self.pool(x)
 query = self.f_pixel(x ).view(batch_size, self.key_channels, -1)
 query = query.permute(0, 2, 1)
 key = self.f_object(proxy ).view(batch_size, self.key_channels, -1)
 value = self.f_down(proxy ).view(batch_size, self.key_channels, -1)
 value = value.permute(0, 2, 1)
 sim_map = torch.matmul(query, key)
 sim_map =(self.key_channels**-.5)* sim_map
sim_map = F.softmax(sim_map, dim=-1)
 context = torch.matmul(sim_map, value)
 context = context.permute(0, 2, 1 ).contiguous()
context = context.view(batch_size, self.key_channels, *x.size() [2:])
 context = self.f_up(context)
 if self.scale > 1:
context = F.interpolate(input=context, size=(h, w), mode='bilinear', align_corners=ALIGN_CORNERS)
 return context
class ObjectAttentionBlock2D(_ObjectAttentionBlock):
def __init__(self,
in_channels,
key_channels,
scale=1,
bn_type=None):
super(ObjectAttentionBlock2D, self ).__init__(in_channels,
key_channels,
scale,
bn_type=bn_type)
 class SpatialOCR_Module(nn.Module):

 def __init__(self,
in_channels,
key_channels,
out_channels,
scale=1,
dropout=0.1,
bn_type=None):
super(SpatialOCR_Module, self ).__init__()
self.object_context_block = ObjectAttentionBlock2D(in_channels,
key_channels,
scale,
bn_type)
 _in_channels = 2 * in_channels
self.conv_bn_dropout = nn.Sequential(
 nn.Conv2d(_in_channels, out_channels, kernel_size=1, padding=0, bias=False),
ModuleHelper.BNReLU(out_channels, bn_type=bn_type),
nn.Dropout2d(dropout)
)
 def forward(self, feats, proxy_feats):
context = self.object_context_block(feats, proxy_feats)
 output = self.conv_bn_dropout(torch.cat([context, feats], 1))
return output
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self ).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = out + residual
out = self.relu(out)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
 self.bn1 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
padding=1, bias=False)
 self.bn2 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1,
bias=False)
 self.bn3 = BatchNorm2d(planes * self.expansion,
momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=relu_inplace)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = out + residual
out = self.relu(out)
 return out
class HighResolutionModule(nn.Module):
def __init__(self, num_branches, blocks, num_blocks, num_inchannels,
num_channels, fuse_method, multi_scale_output=True):
super(HighResolutionModule, self ).__init__()
self._check_branches(
 num_branches, blocks, num_blocks, num_inchannels, num_channels)
 self.num_inchannels = num_inchannels
self.fuse_method = fuse_method
self.num_branches = num_branches
self.multi_scale_output = multi_scale_output
self.branches = self._make_branches(
 num_branches, blocks, num_blocks, num_channels)
 self.fuse_layers = self._make_fuse_layers()
self.relu = nn.ReLU(inplace=relu_inplace)
 def _check_branches(self, num_branches, blocks, num_blocks,
num_inchannels, num_channels):
if num_branches != len(num_blocks):
error_msg = 'NUM_BRANCHES({})<> NUM_BLOCKS({})'.format(
 num_branches, len(num_blocks))
raise ValueError(error_msg)
 if num_branches != len(num_channels):
error_msg = 'NUM_BRANCHES({})<> NUM_CHANNELS({})'.format(
 num_branches, len(num_channels))
raise ValueError(error_msg)
 if num_branches != len(num_inchannels):
error_msg = 'NUM_BRANCHES({})<> NUM_INCHANNELS({})'.format(
 num_branches, len(num_inchannels))
raise ValueError(error_msg)
 def _make_one_branch(self, branch_index, block, num_blocks, num_channels,
stride=1):
downsample = None
if stride != 1 or \
self.num_inchannels[branch_index] != num_channels[branch_index] * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.num_inchannels[branch_index],
num_channels[branch_index] * block.expansion,
kernel_size=1, stride=stride, bias=False),
BatchNorm2d(num_channels[branch_index] * block.expansion,
momentum=BN_MOMENTUM), 
)
 layers = []
layers.append(block(self.num_inchannels[branch_index],
num_channels[branch_index], stride, downsample))
self.num_inchannels[branch_index] = \
num_channels[branch_index] * block.expansion
for i in range(1, num_blocks[branch_index]):
layers.append(block(self.num_inchannels[branch_index],
num_channels[branch_index]))
return nn.Sequential(*layers)
 def _make_branches(self, num_branches, block, num_blocks, num_channels):
branches = []
for i in range(num_branches):
branches.append(
 self._make_one_branch(i, block, num_blocks, num_channels))
return nn.ModuleList(branches)
 def _make_fuse_layers(self):
if self.num_branches == 1:
return None
num_branches = self.num_branches
num_inchannels = self.num_inchannels
fuse_layers = []
for i in range(num_branches if self.multi_scale_output else 1):
fuse_layer = []
for j in range(num_branches):
if j > i:
fuse_layer.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_inchannels[i],
1,
1,
0,
bias=False),
BatchNorm2d(num_inchannels[i], momentum=BN_MOMENTUM)))
 elif j == i:
fuse_layer.append(None)
 else:
conv3x3s = []
for k in range(i-j):
if k == i - j - 1:
num_outchannels_conv3x3 = num_inchannels[i]
conv3x3s.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_outchannels_conv3x3,
3, 2, 1, bias=False),
BatchNorm2d(num_outchannels_conv3x3,
momentum=BN_MOMENTUM)))
 else:
num_outchannels_conv3x3 = num_inchannels[j]
conv3x3s.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_outchannels_conv3x3,
3, 2, 1, bias=False),
BatchNorm2d(num_outchannels_conv3x3,
momentum=BN_MOMENTUM),
nn.ReLU(inplace=relu_inplace)))
 fuse_layer.append(nn.Sequential(*conv3x3s))
fuse_layers.append(nn.ModuleList(fuse_layer))
return nn.ModuleList(fuse_layers)
 def get_num_inchannels(self):
return self.num_inchannels
def forward(self, x):
if self.num_branches == 1:
return [self.branches[0](x[0])]
for i in range(self.num_branches):
x[i] = self.branches[i](x[i])
 x_fuse = []
for i in range(len(self.fuse_layers)) :
y = x[0] if i == 0 else self.fuse_layers[i][0](x[0])
 for j in range(1, self.num_branches):
if i == j:
y = y + x[j]
elif j > i:
width_output = x[i].shape[-1]
height_output = x[i].shape[-2]
y = y + F.interpolate(
 self.fuse_layers[i][j](x[j]),
size=[height_output, width_output],
mode='bilinear', align_corners=ALIGN_CORNERS)
 else:
y = y + self.fuse_layers[i][j](x[j])
 x_fuse.append(self.relu(y))
return x_fuse
blocks_dict = {
'BASIC': BasicBlock,
'BOTTLENECK': Bottleneck
}
class HighResolutionOCRNet(nn.Module):
def __init__(self, config, **kwargs):
global ALIGN_CORNERS
extra = config['MODEL']['EXTRA']
ocr_cfg = config['MODEL']['OCR']
super(HighResolutionOCRNet, self ).__init__()
ALIGN_CORNERS = True
self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1,
bias=False)
 self.bn1 = BatchNorm2d(64, momentum=BN_MOMENTUM)
 self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=2, padding=1,
bias=False)
 self.bn2 = BatchNorm2d(64, momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=relu_inplace)
 self.stage1_cfg = extra['STAGE1']
num_channels = self.stage1_cfg['NUM_CHANNELS'][0]
block = blocks_dict[self.stage1_cfg['BLOCK']]
num_blocks = self.stage1_cfg['NUM_BLOCKS'][0]
self.layer1 = self._make_layer(block, 64, num_channels, num_blocks)
 stage1_out_channel = block.expansion*num_channels
self.stage2_cfg = extra['STAGE2']
num_channels = self.stage2_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage2_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition1 = self._make_transition_layer(
 [stage1_out_channel], num_channels)
 self.stage2, pre_stage_channels = self._make_stage(
 self.stage2_cfg, num_channels)
 self.stage3_cfg = extra['STAGE3']
num_channels = self.stage3_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage3_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition2 = self._make_transition_layer(
 pre_stage_channels, num_channels)
 self.stage3, pre_stage_channels = self._make_stage(
 self.stage3_cfg, num_channels)
 self.stage4_cfg = extra['STAGE4']
num_channels = self.stage4_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage4_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition3 = self._make_transition_layer(
 pre_stage_channels, num_channels)
 self.stage4, pre_stage_channels = self._make_stage(
 self.stage4_cfg, num_channels, multi_scale_output=True)
 last_inp_channels = np.int(np.sum(pre_stage_channels))
ocr_mid_channels = ocr_cfg['MID_CHANNELS']
ocr_key_channels = ocr_cfg['KEY_CHANNELS']
self.conv3x3_ocr = nn.Sequential(
 nn.Conv2d(last_inp_channels, ocr_mid_channels,
kernel_size=3, stride=1, padding=1),
BatchNorm2d(ocr_mid_channels),
nn.ReLU(inplace=relu_inplace), 
)
 self.ocr_gather_head = SpatialGather_Module(config['DATASET']['NUM_CLASSES'])
 self.ocr_distri_head = SpatialOCR_Module(in_channels=ocr_mid_channels,
key_channels=ocr_key_channels,
out_channels=ocr_mid_channels,
scale=1,
dropout=0.05, 
)
 self.cls_head = nn.Conv2d(
 ocr_mid_channels, config['DATASET']['NUM_CLASSES'], kernel_size=1, stride=1, padding=0, bias=True)
 self.aux_head = nn.Sequential(
 nn.Conv2d(last_inp_channels, last_inp_channels,
kernel_size=1, stride=1, padding=0),
BatchNorm2d(last_inp_channels),
nn.ReLU(inplace=relu_inplace),
nn.Conv2d(last_inp_channels, config['DATASET']['NUM_CLASSES'],
kernel_size=1, stride=1, padding=0, bias=True)
)
 self.backbone = nn.ModuleList([self.conv1, self.bn1, self.conv2, self.bn2, self.layer1, self.transition1,
self.stage2, self.transition2, self.stage3, self.transition3, self.stage4])
 self.fuse = nn.ModuleList([self.aux_head, self.conv3x3_ocr, self.ocr_gather_head, self.ocr_distri_head, self.cls_head])
 def _make_transition_layer(
 self, num_channels_pre_layer, num_channels_cur_layer):
num_branches_cur = len(num_channels_cur_layer)
 num_branches_pre = len(num_channels_pre_layer)
 transition_layers = []
for i in range(num_branches_cur):
if i < num_branches_pre:
if num_channels_cur_layer[i] != num_channels_pre_layer[i]:
transition_layers.append(nn.Sequential(
 nn.Conv2d(num_channels_pre_layer[i],
num_channels_cur_layer[i],
3,
1,
1,
bias=False),
BatchNorm2d(
 num_channels_cur_layer[i], momentum=BN_MOMENTUM),
nn.ReLU(inplace=relu_inplace)))
 else:
transition_layers.append(None)
 else:
conv3x3s = []
for j in range(i+1-num_branches_pre):
inchannels = num_channels_pre_layer[-1]
outchannels = num_channels_cur_layer[i] \
if j == i-num_branches_pre else inchannels
conv3x3s.append(nn.Sequential(
 nn.Conv2d(
 inchannels, outchannels, 3, 2, 1, bias=False),
BatchNorm2d(outchannels, momentum=BN_MOMENTUM),
nn.ReLU(inplace=relu_inplace)))
 transition_layers.append(nn.Sequential(*conv3x3s))
return nn.ModuleList(transition_layers)
 def _make_layer(self, block, inplanes, planes, blocks, stride=1):
downsample = None
if stride != 1 or inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False),
BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM), 
)
 layers = []
layers.append(block(inplanes, planes, stride, downsample))
inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(inplanes, planes))
return nn.Sequential(*layers)
 def _make_stage(self, layer_config, num_inchannels,
multi_scale_output=True):
num_modules = layer_config['NUM_MODULES']
num_branches = layer_config['NUM_BRANCHES']
num_blocks = layer_config['NUM_BLOCKS']
num_channels = layer_config['NUM_CHANNELS']
block = blocks_dict[layer_config['BLOCK']]
fuse_method = layer_config['FUSE_METHOD']
modules = []
for i in range(num_modules):
if not multi_scale_output and i == num_modules - 1:
reset_multi_scale_output = False
else:
reset_multi_scale_output = True
modules.append(
 HighResolutionModule(num_branches,
block,
num_blocks,
num_inchannels,
num_channels,
fuse_method,
reset_multi_scale_output)
)
 num_inchannels = modules[-1].get_num_inchannels()
return nn.Sequential(*modules), num_inchannels
def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.conv2(x)
 x = self.bn2(x)
 x = self.relu(x)
 x = self.layer1(x)
 x_list = []
for i in range(self.stage2_cfg['NUM_BRANCHES']):
if self.transition1[i] is not None:
x_list.append(self.transition1[i](x))
else:
x_list.append(x)
 y_list = self.stage2(x_list)
 x_list = []
for i in range(self.stage3_cfg['NUM_BRANCHES']):
if self.transition2[i] is not None:
if i < self.stage2_cfg['NUM_BRANCHES']:
x_list.append(self.transition2[i](y_list[i]))
else:
x_list.append(self.transition2[i](y_list[-1]))
else:
x_list.append(y_list[i])
 y_list = self.stage3(x_list)
 x_list = []
for i in range(self.stage4_cfg['NUM_BRANCHES']):
if self.transition3[i] is not None:
if i < self.stage3_cfg['NUM_BRANCHES']:
x_list.append(self.transition3[i](y_list[i]))
else:
x_list.append(self.transition3[i](y_list[-1]))
else:
x_list.append(y_list[i])
 x = self.stage4(x_list)
 x0_h, x0_w = x[0].size(2), x[0].size(3)
 x1 = F.interpolate(x[1], size=(x0_h, x0_w),
mode='bilinear', align_corners=ALIGN_CORNERS)
 x2 = F.interpolate(x[2], size=(x0_h, x0_w),
mode='bilinear', align_corners=ALIGN_CORNERS)
 x3 = F.interpolate(x[3], size=(x0_h, x0_w),
mode='bilinear', align_corners=ALIGN_CORNERS)
 feats = torch.cat([x[0], x1, x2, x3], 1)
 out_aux_seg = []
out_aux = self.aux_head(feats)
 feats = self.conv3x3_ocr(feats)
 context = self.ocr_gather_head(feats, out_aux)
 feats = self.ocr_distri_head(feats, context)
 out = self.cls_head(feats)
 out_aux_seg.append(out_aux)
 out_aux_seg.append(out)
 return out_aux_seg
def init_weights(self, pretrained='',):
print('=> init weights from normal distribution')
 for name, m in self.named_modules() :
if any(part in name for part in {'cls', 'aux', 'ocr'}):
continue
if isinstance(m, nn.Conv2d):
nn.init.normal_(m.weight, std=0.001)
 elif isinstance(m, BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 if os.path.isfile(pretrained):
pretrained_dict = torch.load(pretrained, map_location={'cuda:0': 'cpu'})
 print('=> loading pretrained model {}'.format(pretrained))
model_dict = self.state_dict()
pretrained_dict = {k.replace('last_layer', 'aux_head' ).replace('model.', ''): v for k, v in pretrained_dict.items() }
pretrained_dict = {k: v for k, v in pretrained_dict.items()
if k in model_dict.keys() }
print('length of pretrained_dict:', len(pretrained_dict.items()))
model_dict.update(pretrained_dict)
 self.load_state_dict(model_dict)
 elif pretrained:
raise RuntimeError('No such file {}'.format(pretrained))
def trainable_parameters(self):
return list(self.backbone.parameters()), list(self.fuse.parameters())
 def get_hr_ocr_seg_model(cfg, **kwargs):
cfg['MODEL']['NUM_CLASSES'] = kwargs['num_classes']
model = HighResolutionOCRNet(cfg, **kwargs)
 return model<define_search_model>
def process_df(df_):
encode_data(df_)
 for f_ in ['v2a1', 'v18q1', 'meaneduc', 'SQBmeaned']:
df_[f_] = df_[f_].fillna(0)
 df_['rez_esc'] = df_['rez_esc'].fillna(0)
 return do_features(df_)
 test = process_df(test)
ALIGN_CORNERS = True
BN_MOMENTUM = 0.1
relu_inplace = True
BatchNorm2d_class = BatchNorm2d = torch.nn.BatchNorm2d
class ModuleHelper:
@staticmethod
def BNReLU(num_features, bn_type=None, **kwargs):
return nn.Sequential(
 BatchNorm2d(num_features, **kwargs),
nn.ReLU() 
)
 @staticmethod
def BatchNorm2d(*args, **kwargs):
return BatchNorm2d
def conv3x3(in_planes, out_planes, stride=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
 class SpatialGather_Module(nn.Module):

 def __init__(self, cls_num=0, scale=1):
super(SpatialGather_Module, self ).__init__()
self.cls_num = cls_num
self.scale = scale
def forward(self, feats, probs):
batch_size, c, h, w = probs.size(0), probs.size(1), probs.size(2), probs.size(3)
 probs = probs.view(batch_size, c, -1)
 feats = feats.view(batch_size, feats.size(1), -1)
 feats = feats.permute(0, 2, 1)
 probs = F.softmax(self.scale * probs, dim=2)
 ocr_context = torch.matmul(probs, feats)\ 
.permute(0, 2, 1 ).unsqueeze(3)
 return ocr_context
class _ObjectAttentionBlock(nn.Module):

 def __init__(self,
in_channels,
key_channels,
scale=1,
bn_type=None):
super(_ObjectAttentionBlock, self ).__init__()
self.scale = scale
self.in_channels = in_channels
self.key_channels = key_channels
self.pool = nn.MaxPool2d(kernel_size=(scale, scale))
self.f_pixel = nn.Sequential(
 nn.Conv2d(in_channels=self.in_channels, out_channels=self.key_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type),
nn.Conv2d(in_channels=self.key_channels, out_channels=self.key_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), 
)
 self.f_object = nn.Sequential(
 nn.Conv2d(in_channels=self.in_channels, out_channels=self.key_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type),
nn.Conv2d(in_channels=self.key_channels, out_channels=self.key_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), 
)
 self.f_down = nn.Sequential(
 nn.Conv2d(in_channels=self.in_channels, out_channels=self.key_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), 
)
 self.f_up = nn.Sequential(
 nn.Conv2d(in_channels=self.key_channels, out_channels=self.in_channels,
kernel_size=1, stride=1, padding=0, bias=False),
ModuleHelper.BNReLU(self.in_channels, bn_type=bn_type), 
)
 def forward(self, x, proxy):
batch_size, h, w = x.size(0), x.size(2), x.size(3)
 if self.scale > 1:
x = self.pool(x)
 query = self.f_pixel(x ).view(batch_size, self.key_channels, -1)
 query = query.permute(0, 2, 1)
 key = self.f_object(proxy ).view(batch_size, self.key_channels, -1)
 value = self.f_down(proxy ).view(batch_size, self.key_channels, -1)
 value = value.permute(0, 2, 1)
 sim_map = torch.matmul(query, key)
 sim_map =(self.key_channels**-.5)* sim_map
sim_map = F.softmax(sim_map, dim=-1)
 context = torch.matmul(sim_map, value)
 context = context.permute(0, 2, 1 ).contiguous()
context = context.view(batch_size, self.key_channels, *x.size() [2:])
 context = self.f_up(context)
 if self.scale > 1:
context = F.interpolate(input=context, size=(h, w), mode='bilinear', align_corners=ALIGN_CORNERS)
 return context
class ObjectAttentionBlock2D(_ObjectAttentionBlock):
def __init__(self,
in_channels,
key_channels,
scale=1,
bn_type=None):
super(ObjectAttentionBlock2D, self ).__init__(in_channels,
key_channels,
scale,
bn_type=bn_type)
 class SpatialOCR_Module(nn.Module):

 def __init__(self,
in_channels,
key_channels,
out_channels,
scale=1,
dropout=0.1,
bn_type=None):
super(SpatialOCR_Module, self ).__init__()
self.object_context_block = ObjectAttentionBlock2D(in_channels,
key_channels,
scale,
bn_type)
 _in_channels = 2 * in_channels
self.conv_bn_dropout = nn.Sequential(
 nn.Conv2d(_in_channels, out_channels, kernel_size=1, padding=0, bias=False),
ModuleHelper.BNReLU(out_channels, bn_type=bn_type),
nn.Dropout2d(dropout)
)
 def forward(self, feats, proxy_feats):
context = self.object_context_block(feats, proxy_feats)
 output = self.conv_bn_dropout(torch.cat([context, feats], 1))
return output
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self ).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = out + residual
out = self.relu(out)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
 self.bn1 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
padding=1, bias=False)
 self.bn2 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1,
bias=False)
 self.bn3 = BatchNorm2d(planes * self.expansion,
momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=relu_inplace)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = out + residual
out = self.relu(out)
 return out
class HighResolutionModule(nn.Module):
def __init__(self, num_branches, blocks, num_blocks, num_inchannels,
num_channels, fuse_method, multi_scale_output=True):
super(HighResolutionModule, self ).__init__()
self._check_branches(
 num_branches, blocks, num_blocks, num_inchannels, num_channels)
 self.num_inchannels = num_inchannels
self.fuse_method = fuse_method
self.num_branches = num_branches
self.multi_scale_output = multi_scale_output
self.branches = self._make_branches(
 num_branches, blocks, num_blocks, num_channels)
 self.fuse_layers = self._make_fuse_layers()
self.relu = nn.ReLU(inplace=relu_inplace)
 def _check_branches(self, num_branches, blocks, num_blocks,
num_inchannels, num_channels):
if num_branches != len(num_blocks):
error_msg = 'NUM_BRANCHES({})<> NUM_BLOCKS({})'.format(
 num_branches, len(num_blocks))
raise ValueError(error_msg)
 if num_branches != len(num_channels):
error_msg = 'NUM_BRANCHES({})<> NUM_CHANNELS({})'.format(
 num_branches, len(num_channels))
raise ValueError(error_msg)
 if num_branches != len(num_inchannels):
error_msg = 'NUM_BRANCHES({})<> NUM_INCHANNELS({})'.format(
 num_branches, len(num_inchannels))
raise ValueError(error_msg)
 def _make_one_branch(self, branch_index, block, num_blocks, num_channels,
stride=1):
downsample = None
if stride != 1 or \
self.num_inchannels[branch_index] != num_channels[branch_index] * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.num_inchannels[branch_index],
num_channels[branch_index] * block.expansion,
kernel_size=1, stride=stride, bias=False),
BatchNorm2d(num_channels[branch_index] * block.expansion,
momentum=BN_MOMENTUM), 
)
 layers = []
layers.append(block(self.num_inchannels[branch_index],
num_channels[branch_index], stride, downsample))
self.num_inchannels[branch_index] = \
num_channels[branch_index] * block.expansion
for i in range(1, num_blocks[branch_index]):
layers.append(block(self.num_inchannels[branch_index],
num_channels[branch_index]))
return nn.Sequential(*layers)
 def _make_branches(self, num_branches, block, num_blocks, num_channels):
branches = []
for i in range(num_branches):
branches.append(
 self._make_one_branch(i, block, num_blocks, num_channels))
return nn.ModuleList(branches)
 def _make_fuse_layers(self):
if self.num_branches == 1:
return None
num_branches = self.num_branches
num_inchannels = self.num_inchannels
fuse_layers = []
for i in range(num_branches if self.multi_scale_output else 1):
fuse_layer = []
for j in range(num_branches):
if j > i:
fuse_layer.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_inchannels[i],
1,
1,
0,
bias=False),
BatchNorm2d(num_inchannels[i], momentum=BN_MOMENTUM)))
 elif j == i:
fuse_layer.append(None)
 else:
conv3x3s = []
for k in range(i-j):
if k == i - j - 1:
num_outchannels_conv3x3 = num_inchannels[i]
conv3x3s.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_outchannels_conv3x3,
3, 2, 1, bias=False),
BatchNorm2d(num_outchannels_conv3x3,
momentum=BN_MOMENTUM)))
 else:
num_outchannels_conv3x3 = num_inchannels[j]
conv3x3s.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_outchannels_conv3x3,
3, 2, 1, bias=False),
BatchNorm2d(num_outchannels_conv3x3,
momentum=BN_MOMENTUM),
nn.ReLU(inplace=relu_inplace)))
 fuse_layer.append(nn.Sequential(*conv3x3s))
fuse_layers.append(nn.ModuleList(fuse_layer))
return nn.ModuleList(fuse_layers)
 def get_num_inchannels(self):
return self.num_inchannels
def forward(self, x):
if self.num_branches == 1:
return [self.branches[0](x[0])]
for i in range(self.num_branches):
x[i] = self.branches[i](x[i])
 x_fuse = []
for i in range(len(self.fuse_layers)) :
y = x[0] if i == 0 else self.fuse_layers[i][0](x[0])
 for j in range(1, self.num_branches):
if i == j:
y = y + x[j]
elif j > i:
width_output = x[i].shape[-1]
height_output = x[i].shape[-2]
y = y + F.interpolate(
 self.fuse_layers[i][j](x[j]),
size=[height_output, width_output],
mode='bilinear', align_corners=ALIGN_CORNERS)
 else:
y = y + self.fuse_layers[i][j](x[j])
 x_fuse.append(self.relu(y))
return x_fuse
blocks_dict = {
'BASIC': BasicBlock,
'BOTTLENECK': Bottleneck
}
class ASPPModule(nn.Module):
def __init__(self, in_channel=720, hidden_channel=256, out_channel=512, dilations=(3, 6, 9), **kwargs):
super(ASPPModule, self ).__init__()
self.asp_conv2 = nn.Sequential(
 nn.Conv2d(in_channel, hidden_channel, kernel_size=1, padding=0, dilation=1, bias=True),
BatchNorm2d(hidden_channel),
nn.ReLU(inplace=relu_inplace),)
 self.asp_conv3 = nn.Sequential(
 nn.Conv2d(in_channel, hidden_channel, kernel_size=3, padding=dilations[0], dilation=dilations[0], bias=True),
BatchNorm2d(hidden_channel),
nn.ReLU(inplace=relu_inplace),)
 self.asp_conv4 = nn.Sequential(
 nn.Conv2d(in_channel, hidden_channel, kernel_size=3, padding=dilations[0], dilation=dilations[0], bias=True),
BatchNorm2d(hidden_channel),
nn.ReLU(inplace=relu_inplace),)
 self.asp_conv5 = nn.Sequential(
 nn.Conv2d(in_channel, hidden_channel, kernel_size=3, padding=dilations[0], dilation=dilations[0], bias=True),
BatchNorm2d(hidden_channel),
nn.ReLU(inplace=relu_inplace),)
 self.asp_fuse = nn.Sequential(
 nn.Conv2d(hidden_channel * 4 + 512, out_channel, kernel_size=1, padding=0, dilation=1, bias=True),
BatchNorm2d(out_channel),
nn.ReLU(inplace=relu_inplace),
nn.Dropout2d(0.1)
)
 def forward(self, feat_hr, feat_context):
feat1 = feat_context
feat2 = self.asp_conv2(feat_hr)
 feat3 = self.asp_conv3(feat_hr)
 feat4 = self.asp_conv4(feat_hr)
 feat5 = self.asp_conv5(feat_hr)
 out = torch.cat(( feat1, feat2, feat3, feat4, feat5), 1)
 output = self.asp_fuse(out)
 return output
class HighResolutionNet_ocr_aspp(nn.Module):
def __init__(self, config, dilations=(3, 6, 9), **kwargs):
global ALIGN_CORNERS
extra = config['MODEL']['EXTRA']
ocr_cfg = config['MODEL']['OCR']
super(HighResolutionNet_ocr_aspp, self ).__init__()
ALIGN_CORNERS = True
self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1,
bias=False)
 self.bn1 = BatchNorm2d(64, momentum=BN_MOMENTUM)
 self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=2, padding=1,
bias=False)
 self.bn2 = BatchNorm2d(64, momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=relu_inplace)
 self.stage1_cfg = extra['STAGE1']
num_channels = self.stage1_cfg['NUM_CHANNELS'][0]
block = blocks_dict[self.stage1_cfg['BLOCK']]
num_blocks = self.stage1_cfg['NUM_BLOCKS'][0]
self.layer1 = self._make_layer(block, 64, num_channels, num_blocks)
 stage1_out_channel = block.expansion * num_channels
self.stage2_cfg = extra['STAGE2']
num_channels = self.stage2_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage2_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition1 = self._make_transition_layer(
 [stage1_out_channel], num_channels)
 self.stage2, pre_stage_channels = self._make_stage(
 self.stage2_cfg, num_channels)
 self.stage3_cfg = extra['STAGE3']
num_channels = self.stage3_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage3_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition2 = self._make_transition_layer(
 pre_stage_channels, num_channels)
 self.stage3, pre_stage_channels = self._make_stage(
 self.stage3_cfg, num_channels)
 self.stage4_cfg = extra['STAGE4']
num_channels = self.stage4_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage4_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition3 = self._make_transition_layer(
 pre_stage_channels, num_channels)
 self.stage4, pre_stage_channels = self._make_stage(
 self.stage4_cfg, num_channels, multi_scale_output=True)
 last_inp_channels = np.int(np.sum(pre_stage_channels))
ocr_mid_channels = ocr_cfg['MID_CHANNELS']
ocr_key_channels = ocr_cfg['KEY_CHANNELS']
self.conv3x3_ocr = nn.Sequential(
 nn.Conv2d(last_inp_channels, ocr_mid_channels,
kernel_size=3, stride=1, padding=1),
BatchNorm2d(ocr_mid_channels),
nn.ReLU(inplace=relu_inplace), 
)
 self.ocr_gather_head = SpatialGather_Module(config['DATASET']['NUM_CLASSES'])
 self.ocr_distri_head = SpatialOCR_Module(in_channels=ocr_mid_channels,
key_channels=ocr_key_channels,
out_channels=ocr_mid_channels,
scale=1,
dropout=0.05, 
)
 self.cls_head = nn.Conv2d(
 ocr_mid_channels, config['DATASET']['NUM_CLASSES'], kernel_size=1, stride=1, padding=0, bias=True)
 self.aux_head = nn.Sequential(
 nn.Conv2d(last_inp_channels, last_inp_channels,
kernel_size=1, stride=1, padding=0),
BatchNorm2d(last_inp_channels),
nn.ReLU(inplace=relu_inplace),
nn.Conv2d(last_inp_channels, config['DATASET']['NUM_CLASSES'],
kernel_size=1, stride=1, padding=0, bias=True)
)
 self.asp_module = ASPPModule(in_channel=720, hidden_channel=256, out_channel=512, dilations=dilations)
 self.backbone = nn.ModuleList([self.conv1, self.bn1, self.conv2, self.bn2, self.layer1, self.transition1,
self.stage2, self.transition2, self.stage3, self.transition3, self.stage4])
 self.fuse = nn.ModuleList(
 [self.aux_head, self.conv3x3_ocr, self.ocr_gather_head, self.ocr_distri_head, self.cls_head,
self.asp_module])
 def _make_transition_layer(
 self, num_channels_pre_layer, num_channels_cur_layer):
num_branches_cur = len(num_channels_cur_layer)
 num_branches_pre = len(num_channels_pre_layer)
 transition_layers = []
for i in range(num_branches_cur):
if i < num_branches_pre:
if num_channels_cur_layer[i] != num_channels_pre_layer[i]:
transition_layers.append(nn.Sequential(
 nn.Conv2d(num_channels_pre_layer[i],
num_channels_cur_layer[i],
3,
1,
1,
bias=False),
BatchNorm2d(
 num_channels_cur_layer[i], momentum=BN_MOMENTUM),
nn.ReLU(inplace=relu_inplace)))
 else:
transition_layers.append(None)
 else:
conv3x3s = []
for j in range(i + 1 - num_branches_pre):
inchannels = num_channels_pre_layer[-1]
outchannels = num_channels_cur_layer[i] \
if j == i - num_branches_pre else inchannels
conv3x3s.append(nn.Sequential(
 nn.Conv2d(
 inchannels, outchannels, 3, 2, 1, bias=False),
BatchNorm2d(outchannels, momentum=BN_MOMENTUM),
nn.ReLU(inplace=relu_inplace)))
 transition_layers.append(nn.Sequential(*conv3x3s))
return nn.ModuleList(transition_layers)
 def _make_layer(self, block, inplanes, planes, blocks, stride=1):
downsample = None
if stride != 1 or inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False),
BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM), 
)
 layers = []
layers.append(block(inplanes, planes, stride, downsample))
inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(inplanes, planes))
return nn.Sequential(*layers)
 def _make_stage(self, layer_config, num_inchannels,
multi_scale_output=True):
num_modules = layer_config['NUM_MODULES']
num_branches = layer_config['NUM_BRANCHES']
num_blocks = layer_config['NUM_BLOCKS']
num_channels = layer_config['NUM_CHANNELS']
block = blocks_dict[layer_config['BLOCK']]
fuse_method = layer_config['FUSE_METHOD']
modules = []
for i in range(num_modules):
if not multi_scale_output and i == num_modules - 1:
reset_multi_scale_output = False
else:
reset_multi_scale_output = True
modules.append(
 HighResolutionModule(num_branches,
block,
num_blocks,
num_inchannels,
num_channels,
fuse_method,
reset_multi_scale_output)
)
 num_inchannels = modules[-1].get_num_inchannels()
return nn.Sequential(*modules), num_inchannels
def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.conv2(x)
 x = self.bn2(x)
 x = self.relu(x)
 x = self.layer1(x)
 x_list = []
for i in range(self.stage2_cfg['NUM_BRANCHES']):
if self.transition1[i] is not None:
x_list.append(self.transition1[i](x))
else:
x_list.append(x)
 y_list = self.stage2(x_list)
 x_list = []
for i in range(self.stage3_cfg['NUM_BRANCHES']):
if self.transition2[i] is not None:
if i < self.stage2_cfg['NUM_BRANCHES']:
x_list.append(self.transition2[i](y_list[i]))
else:
x_list.append(self.transition2[i](y_list[-1]))
else:
x_list.append(y_list[i])
 y_list = self.stage3(x_list)
 x_list = []
for i in range(self.stage4_cfg['NUM_BRANCHES']):
if self.transition3[i] is not None:
if i < self.stage3_cfg['NUM_BRANCHES']:
x_list.append(self.transition3[i](y_list[i]))
else:
x_list.append(self.transition3[i](y_list[-1]))
else:
x_list.append(y_list[i])
 x = self.stage4(x_list)
 x0_h, x0_w = x[0].size(2), x[0].size(3)
 x1 = F.interpolate(x[1], size=(x0_h, x0_w),
mode='bilinear', align_corners=ALIGN_CORNERS)
 x2 = F.interpolate(x[2], size=(x0_h, x0_w),
mode='bilinear', align_corners=ALIGN_CORNERS)
 x3 = F.interpolate(x[3], size=(x0_h, x0_w),
mode='bilinear', align_corners=ALIGN_CORNERS)
 feats_hr = torch.cat([x[0], x1, x2, x3], 1)
 out_aux_seg = []
out_aux = self.aux_head(feats_hr)
 feats = self.conv3x3_ocr(feats_hr)
 context = self.ocr_gather_head(feats, out_aux)
 feats_context = self.ocr_distri_head(feats, context)
 feats = self.asp_module(feats_hr, feats_context)
 out = self.cls_head(feats)
 out_aux_seg.append(out_aux)
 out_aux_seg.append(out)
 return out_aux_seg
def init_weights(self, pretrained='',):
print('=> init weights from normal distribution')
 for name, m in self.named_modules() :
if any(part in name for part in {'cls', 'aux', 'ocr'}):
continue
if isinstance(m, nn.Conv2d):
nn.init.normal_(m.weight, std=0.001)
 elif isinstance(m, BatchNorm2d_class):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 if os.path.isfile(pretrained):
pretrained_dict = torch.load(pretrained, map_location={'cuda:0': 'cpu'})
 print('=> loading pretrained model {}'.format(pretrained))
model_dict = self.state_dict()
pretrained_dict = {k.replace('last_layer', 'aux_head' ).replace('model.', ''): v for k, v in
pretrained_dict.items() }
pretrained_dict = {k: v for k, v in pretrained_dict.items()
if k in model_dict.keys() }
print('length of pretrained_dict:', len(pretrained_dict.items()))
model_dict.update(pretrained_dict)
 self.load_state_dict(model_dict)
 elif pretrained:
raise RuntimeError('No such file {}'.format(pretrained))
def trainable_parameters(self):
return list(self.backbone.parameters()), list(self.fuse.parameters())
 def get_hr_ocr_asp_seg_model(cfg, **kwargs):
cfg['MODEL']['NUM_CLASSES'] = kwargs['num_classes']
model = HighResolutionNet_ocr_aspp(cfg, **kwargs)
 return model 
<train_on_grid>
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_
def INF(B, H, W):
return -torch.diag(torch.tensor(float("inf")).cuda().repeat(H), 0 ).unsqueeze(0 ).repeat(B * W, 1, 1)
 class CC_module(nn.Module):
def __init__(self, in_dim):
super(CC_module, self ).__init__()
self.query_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim // 8, kernel_size=1)
 self.key_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim // 8, kernel_size=1)
 self.value_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1)
 self.softmax = Softmax(dim=3)
 self.INF = INF
self.gamma = nn.Parameter(torch.zeros(1))
def forward(self, x):
m_batchsize, _, height, width = x.size()
proj_query = self.query_conv(x)
 proj_query_H = proj_query.permute(0, 3, 1, 2 ).contiguous().view(m_batchsize * width, -1, height ).permute(0, 2,
1)
 proj_query_W = proj_query.permute(0, 2, 1, 3 ).contiguous().view(m_batchsize * height, -1, width ).permute(0, 2,
1)
 proj_key = self.key_conv(x)
 proj_key_H = proj_key.permute(0, 3, 1, 2 ).contiguous().view(m_batchsize * width, -1, height)
 proj_key_W = proj_key.permute(0, 2, 1, 3 ).contiguous().view(m_batchsize * height, -1, width)
 proj_value = self.value_conv(x)
 proj_value_H = proj_value.permute(0, 3, 1, 2 ).contiguous().view(m_batchsize * width, -1, height)
 proj_value_W = proj_value.permute(0, 2, 1, 3 ).contiguous().view(m_batchsize * height, -1, width)
 energy_H =(torch.bmm(proj_query_H, proj_key_H)+ self.INF(m_batchsize, height, width)).view(m_batchsize, width,
height,
height ).permute(0,
2,
1,
3)
 energy_W = torch.bmm(proj_query_W, proj_key_W ).view(m_batchsize, height, width, width)
 concate = self.softmax(torch.cat([energy_H, energy_W], 3))
att_H = concate[:, :, :, 0:height].permute(0, 2, 1, 3 ).contiguous().view(m_batchsize * width, height, height)
 att_W = concate[:, :, :, height:height + width].contiguous().view(m_batchsize * height, width, width)
 out_H = torch.bmm(proj_value_H, att_H.permute(0, 2, 1)).view(m_batchsize, width, -1, height ).permute(0, 2, 3, 1)
 out_W = torch.bmm(proj_value_W, att_W.permute(0, 2, 1)).view(m_batchsize, height, -1, width ).permute(0, 2, 1, 3)
 return self.gamma *(out_H + out_W)+ x
class RCCAModule(nn.Module):
def __init__(self, in_channels, out_channels, num_classes):
super(RCCAModule, self ).__init__()
inter_channels = in_channels // 4
self.conva = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
nn.BatchNorm2d(inter_channels))
self.cca = CC_module(inter_channels)
 self.convb = nn.Sequential(nn.Conv2d(inter_channels, inter_channels, 3, padding=1, bias=False),
nn.BatchNorm2d(inter_channels))
self.bottleneck = nn.Sequential(
 nn.Conv2d(in_channels+inter_channels, out_channels, kernel_size=3, padding=1, dilation=1, bias=False),
nn.BatchNorm2d(out_channels),
nn.Dropout2d(0.1),
nn.Conv2d(512, num_classes, kernel_size=1, stride=1, padding=0, bias=True)
)
 def forward(self, x, recurrence=1):
output = self.conva(x)
 for i in range(recurrence):
output = self.cca(output)
 output = self.convb(output)
 output = self.bottleneck(torch.cat([x, output], 1))
return output
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self ).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=relu_inplace)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = out + residual
out = self.relu(out)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
 self.bn1 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
padding=1, bias=False)
 self.bn2 = BatchNorm2d(planes, momentum=BN_MOMENTUM)
 self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1,
bias=False)
 self.bn3 = BatchNorm2d(planes * self.expansion,
momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=relu_inplace)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = out + residual
out = self.relu(out)
 return out
class HighResolutionModule(nn.Module):
def __init__(self, num_branches, blocks, num_blocks, num_inchannels,
num_channels, fuse_method, multi_scale_output=True):
super(HighResolutionModule, self ).__init__()
self._check_branches(
 num_branches, blocks, num_blocks, num_inchannels, num_channels)
 self.num_inchannels = num_inchannels
self.fuse_method = fuse_method
self.num_branches = num_branches
self.multi_scale_output = multi_scale_output
self.branches = self._make_branches(
 num_branches, blocks, num_blocks, num_channels)
 self.fuse_layers = self._make_fuse_layers()
self.relu = nn.ReLU(inplace=relu_inplace)
 def _check_branches(self, num_branches, blocks, num_blocks,
num_inchannels, num_channels):
if num_branches != len(num_blocks):
error_msg = 'NUM_BRANCHES({})<> NUM_BLOCKS({})'.format(
 num_branches, len(num_blocks))
raise ValueError(error_msg)
 if num_branches != len(num_channels):
error_msg = 'NUM_BRANCHES({})<> NUM_CHANNELS({})'.format(
 num_branches, len(num_channels))
raise ValueError(error_msg)
 if num_branches != len(num_inchannels):
error_msg = 'NUM_BRANCHES({})<> NUM_INCHANNELS({})'.format(
 num_branches, len(num_inchannels))
raise ValueError(error_msg)
 def _make_one_branch(self, branch_index, block, num_blocks, num_channels,
stride=1):
downsample = None
if stride != 1 or \
self.num_inchannels[branch_index] != num_channels[branch_index] * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.num_inchannels[branch_index],
num_channels[branch_index] * block.expansion,
kernel_size=1, stride=stride, bias=False),
BatchNorm2d(num_channels[branch_index] * block.expansion,
momentum=BN_MOMENTUM), 
)
 layers = []
layers.append(block(self.num_inchannels[branch_index],
num_channels[branch_index], stride, downsample))
self.num_inchannels[branch_index] = \
num_channels[branch_index] * block.expansion
for i in range(1, num_blocks[branch_index]):
layers.append(block(self.num_inchannels[branch_index],
num_channels[branch_index]))
return nn.Sequential(*layers)
 def _make_branches(self, num_branches, block, num_blocks, num_channels):
branches = []
for i in range(num_branches):
branches.append(
 self._make_one_branch(i, block, num_blocks, num_channels))
return nn.ModuleList(branches)
 def _make_fuse_layers(self):
if self.num_branches == 1:
return None
num_branches = self.num_branches
num_inchannels = self.num_inchannels
fuse_layers = []
for i in range(num_branches if self.multi_scale_output else 1):
fuse_layer = []
for j in range(num_branches):
if j > i:
fuse_layer.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_inchannels[i],
1,
1,
0,
bias=False),
BatchNorm2d(num_inchannels[i], momentum=BN_MOMENTUM)))
 elif j == i:
fuse_layer.append(None)
 else:
conv3x3s = []
for k in range(i - j):
if k == i - j - 1:
num_outchannels_conv3x3 = num_inchannels[i]
conv3x3s.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_outchannels_conv3x3,
3, 2, 1, bias=False),
BatchNorm2d(num_outchannels_conv3x3,
momentum=BN_MOMENTUM)))
 else:
num_outchannels_conv3x3 = num_inchannels[j]
conv3x3s.append(nn.Sequential(
 nn.Conv2d(num_inchannels[j],
num_outchannels_conv3x3,
3, 2, 1, bias=False),
BatchNorm2d(num_outchannels_conv3x3,
momentum=BN_MOMENTUM),
nn.ReLU(inplace=relu_inplace)))
 fuse_layer.append(nn.Sequential(*conv3x3s))
fuse_layers.append(nn.ModuleList(fuse_layer))
return nn.ModuleList(fuse_layers)
 def get_num_inchannels(self):
return self.num_inchannels
def forward(self, x):
if self.num_branches == 1:
return [self.branches[0](x[0])]
for i in range(self.num_branches):
x[i] = self.branches[i](x[i])
 x_fuse = []
for i in range(len(self.fuse_layers)) :
y = x[0] if i == 0 else self.fuse_layers[i][0](x[0])
 for j in range(1, self.num_branches):
if i == j:
y = y + x[j]
elif j > i:
width_output = x[i].shape[-1]
height_output = x[i].shape[-2]
y = y + F.interpolate(
 self.fuse_layers[i][j](x[j]),
size=[height_output, width_output],
mode='bilinear', align_corners=ALIGN_CORNERS)
 else:
y = y + self.fuse_layers[i][j](x[j])
 x_fuse.append(self.relu(y))
return x_fuse
blocks_dict = {
'BASIC': BasicBlock,
'BOTTLENECK': Bottleneck
}
class HighResolutionNetCC(nn.Module):
def __init__(self, config, **kwargs):
global ALIGN_CORNERS
extra = config['MODEL']['EXTRA']
ocr_cfg = config['MODEL']['OCR']
super(HighResolutionNetCC, self ).__init__()
ALIGN_CORNERS = True
self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1,
bias=False)
 self.bn1 = BatchNorm2d(64, momentum=BN_MOMENTUM)
 self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=2, padding=1,
bias=False)
 self.bn2 = BatchNorm2d(64, momentum=BN_MOMENTUM)
 self.relu = nn.ReLU(inplace=relu_inplace)
 self.stage1_cfg = extra['STAGE1']
num_channels = self.stage1_cfg['NUM_CHANNELS'][0]
block = blocks_dict[self.stage1_cfg['BLOCK']]
num_blocks = self.stage1_cfg['NUM_BLOCKS'][0]
self.layer1 = self._make_layer(block, 64, num_channels, num_blocks)
 stage1_out_channel = block.expansion * num_channels
self.stage2_cfg = extra['STAGE2']
num_channels = self.stage2_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage2_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition1 = self._make_transition_layer(
 [stage1_out_channel], num_channels)
 self.stage2, pre_stage_channels = self._make_stage(
 self.stage2_cfg, num_channels)
 self.stage3_cfg = extra['STAGE3']
num_channels = self.stage3_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage3_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition2 = self._make_transition_layer(
 pre_stage_channels, num_channels)
 self.stage3, pre_stage_channels = self._make_stage(
 self.stage3_cfg, num_channels)
 self.stage4_cfg = extra['STAGE4']
num_channels = self.stage4_cfg['NUM_CHANNELS']
block = blocks_dict[self.stage4_cfg['BLOCK']]
num_channels = [
num_channels[i] * block.expansion for i in range(len(num_channels)) ]
self.transition3 = self._make_transition_layer(
 pre_stage_channels, num_channels)
 self.stage4, pre_stage_channels = self._make_stage(
 self.stage4_cfg, num_channels, multi_scale_output=True)
 last_inp_channels = np.int(np.sum(pre_stage_channels))
ocr_mid_channels = ocr_cfg['MID_CHANNELS']
ocr_key_channels = ocr_cfg['KEY_CHANNELS']
self.conv3x3_ocr = nn.Sequential(
 nn.Conv2d(last_inp_channels, ocr_mid_channels,
kernel_size=3, stride=1, padding=1),
BatchNorm2d(ocr_mid_channels),
nn.ReLU(inplace=relu_inplace), 
)
 self.cc = RCCAModule(512, 512, 5)
 self.aux_head = nn.Sequential(
 nn.Conv2d(last_inp_channels, last_inp_channels,
kernel_size=1, stride=1, padding=0),
BatchNorm2d(last_inp_channels),
nn.ReLU(inplace=relu_inplace),
nn.Conv2d(last_inp_channels, config['DATASET']['NUM_CLASSES'],
kernel_size=1, stride=1, padding=0, bias=True)
)
 self.backbone = nn.ModuleList([self.conv1, self.bn1, self.conv2, self.bn2, self.layer1, self.transition1,
self.stage2, self.transition2, self.stage3, self.transition3, self.stage4])
 self.fuse = nn.ModuleList(
 [self.aux_head, self.conv3x3_ocr, self.cc])
 def _make_transition_layer(
 self, num_channels_pre_layer, num_channels_cur_layer):
num_branches_cur = len(num_channels_cur_layer)
 num_branches_pre = len(num_channels_pre_layer)
 transition_layers = []
for i in range(num_branches_cur):
if i < num_branches_pre:
if num_channels_cur_layer[i] != num_channels_pre_layer[i]:
transition_layers.append(nn.Sequential(
 nn.Conv2d(num_channels_pre_layer[i],
num_channels_cur_layer[i],
3,
1,
1,
bias=False),
BatchNorm2d(
 num_channels_cur_layer[i], momentum=BN_MOMENTUM),
nn.ReLU(inplace=relu_inplace)))
 else:
transition_layers.append(None)
 else:
conv3x3s = []
for j in range(i + 1 - num_branches_pre):
inchannels = num_channels_pre_layer[-1]
outchannels = num_channels_cur_layer[i] \
if j == i - num_branches_pre else inchannels
conv3x3s.append(nn.Sequential(
 nn.Conv2d(
 inchannels, outchannels, 3, 2, 1, bias=False),
BatchNorm2d(outchannels, momentum=BN_MOMENTUM),
nn.ReLU(inplace=relu_inplace)))
 transition_layers.append(nn.Sequential(*conv3x3s))
return nn.ModuleList(transition_layers)
 def _make_layer(self, block, inplanes, planes, blocks, stride=1):
downsample = None
if stride != 1 or inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False),
BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM), 
)
 layers = []
layers.append(block(inplanes, planes, stride, downsample))
inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(inplanes, planes))
return nn.Sequential(*layers)
 def _make_stage(self, layer_config, num_inchannels,
multi_scale_output=True):
num_modules = layer_config['NUM_MODULES']
num_branches = layer_config['NUM_BRANCHES']
num_blocks = layer_config['NUM_BLOCKS']
num_channels = layer_config['NUM_CHANNELS']
block = blocks_dict[layer_config['BLOCK']]
fuse_method = layer_config['FUSE_METHOD']
modules = []
for i in range(num_modules):
if not multi_scale_output and i == num_modules - 1:
reset_multi_scale_output = False
else:
reset_multi_scale_output = True
modules.append(
 HighResolutionModule(num_branches,
block,
num_blocks,
num_inchannels,
num_channels,
fuse_method,
reset_multi_scale_output)
)
 num_inchannels = modules[-1].get_num_inchannels()
return nn.Sequential(*modules), num_inchannels
def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.conv2(x)
 x = self.bn2(x)
 x = self.relu(x)
 x = self.layer1(x)
 x_list = []
for i in range(self.stage2_cfg['NUM_BRANCHES']):
if self.transition1[i] is not None:
x_list.append(self.transition1[i](x))
else:
x_list.append(x)
 y_list = self.stage2(x_list)
 x_list = []
for i in range(self.stage3_cfg['NUM_BRANCHES']):
if self.transition2[i] is not None:
if i < self.stage2_cfg['NUM_BRANCHES']:
x_list.append(self.transition2[i](y_list[i]))
else:
x_list.append(self.transition2[i](y_list[-1]))
else:
x_list.append(y_list[i])
 y_list = self.stage3(x_list)
 x_list = []
for i in range(self.stage4_cfg['NUM_BRANCHES']):
if self.transition3[i] is not None:
if i < self.stage3_cfg['NUM_BRANCHES']:
x_list.append(self.transition3[i](y_list[i]))
else:
x_list.append(self.transition3[i](y_list[-1]))
else:
x_list.append(y_list[i])
 x = self.stage4(x_list)
 x0_h, x0_w = x[0].size(2), x[0].size(3)
 x1 = F.interpolate(x[1], size=(x0_h, x0_w),
mode='bilinear', align_corners=ALIGN_CORNERS)
 x2 = F.interpolate(x[2], size=(x0_h, x0_w),
mode='bilinear', align_corners=ALIGN_CORNERS)
 x3 = F.interpolate(x[3], size=(x0_h, x0_w),
mode='bilinear', align_corners=ALIGN_CORNERS)
 feats = torch.cat([x[0], x1, x2, x3], 1)
 out_aux_seg = []
out_aux = self.aux_head(feats)
 feats = self.conv3x3_ocr(feats)
 out = self.cc(feats)
 out_aux_seg.append(out_aux)
 out_aux_seg.append(out)
 return out_aux_seg
def init_weights(self, pretrained='',):
print('=> init weights from normal distribution')
 for name, m in self.named_modules() :
if any(part in name for part in {'cls', 'aux', 'ocr'}):
continue
if isinstance(m, nn.Conv2d):
nn.init.normal_(m.weight, std=0.001)
 elif isinstance(m, BatchNorm2d_class):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 if os.path.isfile(pretrained):
pretrained_dict = torch.load(pretrained, map_location={'cuda:0': 'cpu'})
 print('=> loading pretrained model {}'.format(pretrained))
model_dict = self.state_dict()
pretrained_dict = {k.replace('last_layer', 'aux_head' ).replace('model.', ''): v for k, v in
pretrained_dict.items() }
pretrained_dict = {k: v for k, v in pretrained_dict.items()
if k in model_dict.keys() }
print('length of pretrained_dict:', len(pretrained_dict.items()))
model_dict.update(pretrained_dict)
 self.load_state_dict(model_dict)
 elif pretrained:
raise RuntimeError('No such file {}'.format(pretrained))
def trainable_parameters(self):
return list(self.backbone.parameters()), list(self.fuse.parameters())
 def get_seg_cc_model(cfg, **kwargs):
cfg['MODEL']['NUM_CLASSES'] = kwargs['num_classes']
model = HighResolutionNetCC(cfg, **kwargs)
 return model 
<import_modules>
train = do_features(poverty_train )
class Bottle2neck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None, baseWidth=26, scale=4, stype='normal'):

 super(Bottle2neck, self ).__init__()
width = int(math.floor(planes *(baseWidth / 64.0)))
 self.conv1 = nn.Conv2d(inplanes, width * scale, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(width * scale)
 if scale == 1:
self.nums = 1
else:
self.nums = scale - 1
if stype == 'stage':
self.pool = nn.AvgPool2d(kernel_size=3, stride=stride, padding=1)
 convs = []
bns = []
for i in range(self.nums):
convs.append(nn.Conv2d(width, width, kernel_size=3, stride=stride, padding=1, bias=False))
bns.append(nn.BatchNorm2d(width))
self.convs = nn.ModuleList(convs)
 self.bns = nn.ModuleList(bns)
 self.conv3 = nn.Conv2d(width * scale, planes * self.expansion, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * self.expansion)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.stype = stype
self.scale = scale
self.width = width
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 spx = torch.split(out, self.width, 1)
 for i, _ in enumerate(range(self.nums)) :
if i == 0 or self.stype == 'stage':
sp = spx[i]
else:
sp = sp + spx[i]
sp = self.convs[i](sp)
 sp = self.relu(self.bns[i](sp))
if i == 0:
out = sp
else:
out = torch.cat(( out, sp), 1)
 if self.scale != 1 and self.stype == 'normal':
out = torch.cat(( out, spx[self.nums]), 1)
 elif self.scale != 1 and self.stype == 'stage':
out = torch.cat(( out, self.pool(spx[self.nums])) , 1)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class Res2Net(nn.Module):
def __init__(self, block, layers, baseWidth=26, scale=4, num_classes=1000):
self.inplanes = 64
super(Res2Net, self ).__init__()
self.baseWidth = baseWidth
self.scale = scale
self.conv1 = nn.Sequential(
 nn.Conv2d(3, 32, 3, 2, 1, bias=False),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.Conv2d(32, 32, 3, 1, 1, bias=False),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.Conv2d(32, 64, 3, 1, 1, bias=False)
)
 self.bn1 = nn.BatchNorm2d(64)
 self.relu = nn.ReLU()
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0])
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
 self.avgpool = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 def _make_layer(self, block, planes, blocks, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.AvgPool2d(kernel_size=stride, stride=stride,
ceil_mode=True, count_include_pad=False),
nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=1, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
stype='stage', baseWidth=self.baseWidth, scale=self.scale))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, baseWidth=self.baseWidth, scale=self.scale))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = x.view(x.size(0), -1)
 x = self.fc(x)
 return x
def res2net101_v1b(pretrained=False, **kwargs):

 model = Res2Net(Bottle2neck, [3, 4, 23, 3], baseWidth=26, scale=4, **kwargs)
 return model<choose_model_class>
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1 )
class Res2NetDeeplabV3P(nn.Module):
def __init__(self, num_classes, aspp_dilate=(12, 24, 36)) :
super(Res2NetDeeplabV3P, self ).__init__()
backbone = res2net101_v1b(True)
 self.conv1 = backbone.conv1
self.bn1 = backbone.bn1
self.relu = backbone.relu
self.maxpool = backbone.maxpool
self.stage1 = backbone.layer1
self.stage2 = backbone.layer2
self.stage3 = backbone.layer3
self.stage4 = backbone.layer4
self.project = nn.Sequential(
 nn.Conv2d(512, 48, 1, bias=False),
nn.BatchNorm2d(48),
nn.ReLU(inplace=True), 
)
 self.aspp = ASPP(2048, aspp_dilate)
 self.classifier = nn.Sequential(
 nn.Conv2d(304, 256, 3, padding=1, bias=False),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True),
nn.Conv2d(256, num_classes, 1)
)
 self.backbone = nn.ModuleList([self.conv1, self.bn1, self.stage1, self.stage2, self.stage3, self.stage4])
 self.fuse = nn.ModuleList([self.project, self.aspp, self.classifier])
 self._init_fuse_weight()
def trainable_parameters(self):
return list(self.backbone.parameters()), list(self.fuse.parameters())
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.stage1(x)
 x = self.stage2(x)
 low_feat = x.clone()
x = self.stage3(x)
 out_feat = self.stage4(x)
 low_level_feature = self.project(low_feat)
 output_feature = self.aspp(out_feat)
 output_feature = F.interpolate(output_feature, size=low_level_feature.shape[2:], mode='bilinear',
align_corners=False)
 return self.classifier(torch.cat([low_level_feature, output_feature], dim=1))
def _init_fuse_weight(self):
for m in self.fuse.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 elif isinstance(m,(nn.BatchNorm2d, nn.GroupNorm)) :
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 class ASPPConv(nn.Sequential):
def __init__(self, in_channels, out_channels, dilation):
modules = [
nn.Conv2d(in_channels, out_channels, 3, padding=dilation, dilation=dilation, bias=False),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True)
 ]
super(ASPPConv, self ).__init__(*modules)
 class ASPPPooling(nn.Sequential):
def __init__(self, in_channels, out_channels):
super(ASPPPooling, self ).__init__(
 nn.AdaptiveAvgPool2d(1),
nn.Conv2d(in_channels, out_channels, 1, bias=False),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True))
def forward(self, x):
size = x.shape[-2:]
x = super(ASPPPooling, self ).forward(x)
 return F.interpolate(x, size=size, mode='bilinear', align_corners=False)
 class ASPP(nn.Module):
def __init__(self, in_channels, atrous_rates):
super(ASPP, self ).__init__()
out_channels = 256
modules = []
modules.append(nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 1, bias=False),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True)))
 rate1, rate2, rate3 = tuple(atrous_rates)
 modules.append(ASPPConv(in_channels, out_channels, rate1))
modules.append(ASPPConv(in_channels, out_channels, rate2))
modules.append(ASPPConv(in_channels, out_channels, rate3))
modules.append(ASPPPooling(in_channels, out_channels))
self.convs = nn.ModuleList(modules)
 self.project = nn.Sequential(
 nn.Conv2d(5 * out_channels, out_channels, 1, bias=False),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True),
nn.Dropout(0.1),)
 def forward(self, x):
res = []
for conv in self.convs:
res.append(conv(x))
res = torch.cat(res, dim=1)
 return self.project(res )<categorify>
cols_2_drop = ['agg18_estadocivil1_MEAN', 'agg18_estadocivil3_COUNT', 'agg18_estadocivil4_COUNT',
'agg18_estadocivil5_COUNT', 'agg18_estadocivil6_COUNT', 'agg18_estadocivil7_COUNT',
'agg18_instlevel1_COUNT', 'agg18_instlevel2_COUNT', 'agg18_instlevel3_COUNT',
'agg18_instlevel4_COUNT', 'agg18_instlevel5_COUNT', 'agg18_instlevel6_COUNT',
'agg18_instlevel7_COUNT', 'agg18_instlevel8_COUNT', 'agg18_instlevel9_COUNT',
'agg18_parentesco10_COUNT', 'agg18_parentesco10_MEAN', 'agg18_parentesco11_COUNT',
'agg18_parentesco11_MEAN', 'agg18_parentesco12_COUNT', 'agg18_parentesco12_MEAN',
'agg18_parentesco1_COUNT', 'agg18_parentesco2_COUNT', 'agg18_parentesco3_COUNT',
'agg18_parentesco4_COUNT', 'agg18_parentesco4_MEAN', 'agg18_parentesco5_COUNT',
'agg18_parentesco6_COUNT', 'agg18_parentesco6_MEAN', 'agg18_parentesco7_COUNT',
'agg18_parentesco7_MEAN', 'agg18_parentesco8_COUNT', 'agg18_parentesco8_MEAN',
'agg18_parentesco9_COUNT', 'fe_people_weird_stat', 'hacapo', 'hacdor', 'mobilephone',
'parentesco1', 'rez_esc', 'v14a', 'v18q',
'agg18_age_MIN', 'agg18_age_MAX' , 'agg18_age_MEAN', 'agg18_escolari_MIN',
'agg18_escolari_MAX', 'agg18_escolari_MEAN', 'agg18_dis_MEAN',
'agg18_estadocivil1_COUNT', 'agg18_estadocivil2_MEAN', 'agg18_estadocivil2_COUNT',
'agg18_estadocivil3_MEAN', 'agg18_estadocivil4_MEAN', 'agg18_estadocivil5_MEAN',
'agg18_estadocivil6_MEAN','agg18_estadocivil7_MEAN','agg18_parentesco1_MEAN',
'agg18_parentesco2_MEAN','agg18_parentesco3_MEAN','agg18_parentesco5_MEAN','agg18_parentesco9_MEAN',
'agg18_instlevel1_MEAN','agg18_instlevel2_MEAN','agg18_instlevel3_MEAN','agg18_instlevel4_MEAN',
'agg18_instlevel5_MEAN','agg18_instlevel6_MEAN','agg18_instlevel7_MEAN','agg18_instlevel8_MEAN',
'agg18_instlevel9_MEAN']
X.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True)
 test.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True )
def mask_to_rle(mask):

 pixels= mask.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<load_pretrained>
from torch.utils.data import DataLoader
from tqdm import tqdm
import cv2 
<install_modules>
param_distribs = {
'penalty': ['l1', 'l2'],
'C': [0.001, 0.01, 0.1, 0.25, 0.50, 0.75, 1, 5,
10, 500, 750, 1000, 1250, 1500, 2000, 10000],
'fit_intercept': [True, False],
'class_weight': [None, 'balanced']
}
logit = LogisticRegression(random_state=42, solver = "liblinear", multi_class = 'ovr',
n_jobs = 4, max_iter = 200)
 rnd_search = GridSearchCV(logit, param_grid=param_distribs,
cv=5, scoring='f1_macro',
verbose=2, n_jobs=4)
 rnd_search.fit(X, y )
!cp -r.. /input/pytorch-segmentation-models/segmentation_models_pytorch-0.1.3-py3-none-any.whl /kaggle/working
!cp -r.. /input/pytorch-segmentation-models/timm-0.3.2-py3-none-any.whl /kaggle/working
!cp -r.. /input/smpwhl/efficientnet_pytorch-0.6.3-py3-none-any.whl /kaggle/working
!cp -r.. /input/smpwhl/pretrainedmodels-0.7.4-py3-none-any.whl /kaggle/working
!pip install /kaggle/working/timm-0.3.2-py3-none-any.whl
!pip install /kaggle/working/efficientnet_pytorch-0.6.3-py3-none-any.whl
!pip install /kaggle/working/pretrainedmodels-0.7.4-py3-none-any.whl
!pip install /kaggle/working/segmentation_models_pytorch-0.1.3-py3-none-any.whl<load_pretrained>
cvres = rnd_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(mean_score, params )
model = smp.Unet(
 encoder_name="timm-efficientnet-b5",
encoder_weights = None,
in_channels=3,
classes=5, 
)
 model_path = '.. /input/u-enetb5-full-all-ep110/enb5_net_dice96.04_ep110.pth'
model = load_model(model, model_path)
 model = model.cuda()<create_dataframe>
print(rnd_search.best_score_, rnd_search.best_estimator_, rnd_search.best_params_ )
defect_root = '/kaggle/input/severstal-steel-defect-detection/test_images'
dataset = SteelDataset(defect_root, resize_long=(800, 800), img_normal=True, to_torch=True)
 data_loader = DataLoader(dataset, batch_size=4, shuffle=False, num_workers=4, pin_memory=True )<define_variables>
device='cuda:0' if torch.cuda.is_available() else 'cpu'
image_id_class_id = []
encoded_pixel = []
threshold_label = [ 0.75, 0.85, 0.50, 0.50,]
threshold_mask_pixel = [ 0.40, 0.40, 0.40, 0.40,]
threshold_mask_size = [ 50, 250, 1400, 1800,]
with torch.no_grad() :
for i_iter, batch in enumerate(data_loader):
img_name = [name.split('/')[-1] for name in batch['name']]
img = batch['img'].to(device)
 if 1:
logit = model(img)
 if isinstance(logit, list):
logit = logit[1]
logit = F.interpolate(logit,(256, 1600), mode='bilinear', align_corners=True)
 probability = torch.softmax(logit,1)
 probability_mask = probability[:,1:]
probability_label = probability_mask_to_probability_label(probability)[:,1:]
num_augment = 1
if 1 :
logit = model(torch.flip(img,dims=[3]))
if isinstance(logit, list):
logit = logit[1]
logit = F.interpolate(logit,(256, 1600), mode='bilinear', align_corners=True)
 probability = torch.softmax(torch.flip(logit,dims=[3]),1)
 probability_mask += probability[:,1:]
probability_label += probability_mask_to_probability_label(probability)[:,1:]
num_augment +=1
if 1 :
logit = model(torch.flip(img,dims=[2]))
if isinstance(logit, list):
logit = logit[1]
logit = F.interpolate(logit,(256, 1600), mode='bilinear', align_corners=True)
 probability = torch.softmax(torch.flip(logit,dims=[2]),1)
 probability_mask += probability[:,1:]
probability_label += probability_mask_to_probability_label(probability)[:,1:]
num_augment +=1
probability_mask = probability_mask/num_augment
probability_label = probability_label/num_augment
probability_mask = probability_mask.data.cpu().numpy()
probability_label = probability_label.data.cpu().numpy()
for b in range(img.shape[0]):
for c in range(4):
rle=''
predict_label = probability_label[b,c]>threshold_label[c]
if predict_label:
try:
predict_mask = probability_mask[b,c] > threshold_mask_pixel[c]
predict_mask = post_process(predict_mask, threshold_mask_size[c])
 rle = run_length_encode(predict_mask)
 except:
print('An exception occurred : %s'%(image_id[b]+'_%d'%(c+1)))
 image_id_class_id.append(img_name[b]+'_%d'%(c+1))
encoded_pixel.append(rle)
 if(i_iter+1)% 100 == 0:
print('{}/{}'.format(i_iter+1,len(dataset)) , 'time:', time.time() -s_time)
 df = pd.DataFrame(zip(image_id_class_id, encoded_pixel), columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv('submission.csv', index=False)
 print('total_time:', time.time() -s_time )<import_modules>
y_subm = pd.read_csv('.. /input/sample_submission.csv')
 y_subm['Target'] = rnd_search.predict(test)+ 1
y_subm.to_csv('submission.csv', index=False )