Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<categorify>
%matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore' )
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<define_search_space>
train = pd.read_csv('.. /input/train.csv')
 train.head()
threshold_pixel = [0.5,0.5,0.5,0.5,]
min_size = [200,1500,1500,2000]<normalization>
test = pd.read_csv('.. /input/test.csv')
 test.tail()
def null_augment(input): return input
def flip_lr_augment(input): return torch.flip(input, dims=[2])
 def flip_ud_augment(input): return torch.flip(input, dims=[3])
 def null_inverse_augment(logit): return logit
def flip_lr_inverse_augment(logit): return torch.flip(logit, dims=[2])
 def flip_ud_inverse_augment(logit): return torch.flip(logit, dims=[3])
 augment =(
(null_augment, null_inverse_augment), 
(flip_lr_augment,flip_lr_inverse_augment), 
(flip_ud_augment,flip_ud_inverse_augment),
)<find_best_params>
data = []
for col in train.columns:
if col == 'target' or col == 'id':
role = col
else:
role = 'input'
if 'bin' in col or col == 'target':
level = 'binary'
elif 'cat' in col or col == 'id':
level = 'nominal'
elif train[col].dtype == float:
level = 'interval'
elif train[col].dtype == int:
level = 'ordinal'
keep = True
if col == 'id':
keep = False
dtype = train[col].dtype
col_dict = {'col_name': col, 'role': role, 'level': level, 'keep': keep, 'dtype': dtype}
data.append(col_dict)
 meta = pd.DataFrame(data, columns=['col_name', 'role', 'level', 'keep', 'dtype'])
 meta.set_index('col_name', inplace=True )
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
images = images.cuda()
batch_preds = 0
probabilities = []
for model in models:
model = model.cuda()
for k,(a, inv_a)in enumerate(augment):
logit = model(a(images))
p = inv_a(torch.sigmoid(logit))
if k ==0:
probability = p**0.5
else:
probability += p**0.5
probability = probability/len(augment)
 probabilities.append(probability)
 batch_preds+=probability
batch_preds = batch_preds.data.cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, threshold_pixel[cls], min_size[cls])
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df_segmentation = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'] )<create_dataframe>
train.isnull().any().any()
df= df_segmentation.copy()<data_type_conversions>
ms_cols = []
train_copy = train
train_copy = train_copy.replace(-1, np.NaN)
 for col in train_copy.columns:
ms_nb = train_copy[col].isnull().sum()
if ms_nb > 0:
ms_cols.append(col)
 print('Column {} has {} records({:.2%})with missing values'.format(col, ms_nb, ms_nb/train_copy.shape[0]))
if 1:
df['Class'] = df['ImageId_ClassId'].str[-1].astype(np.int32)
 df['Label'] =(df['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df['Class']==1)&(df['Label']==1)).sum()
pos2 =(( df['Class']==2)&(df['Label']==1)).sum()
pos3 =(( df['Class']==3)&(df['Label']==1)).sum()
pos4 =(( df['Class']==4)&(df['Label']==1)).sum()
num_image = len(df)//4
num = len(df)
 pos =(df['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))<define_variables>
drop_cols = ['ps_car_03_cat', 'ps_car_05_cat']
real_train = train.drop(drop_cols, axis=1)
 real_test = test.drop(drop_cols, axis=1)
 meta.loc[drop_cols, 'keep'] = False
mean_imp = Imputer(missing_values=-1, strategy='mean', axis=0)
 mode_imp = Imputer(missing_values=-1, strategy='most_frequent', axis=0)
 real_train['ps_reg_03'] = mean_imp.fit_transform(X=real_train[['ps_reg_03']] ).ravel()
real_train['ps_car_11'] = mode_imp.fit_transform(X=real_train[['ps_car_11']] ).ravel()
real_train['ps_car_12'] = mode_imp.fit_transform(X=real_train[['ps_car_12']] ).ravel()
real_train['ps_car_14'] = mean_imp.fit_transform(X=real_train[['ps_car_14']] ).ravel()
real_test['ps_reg_03'] = mean_imp.fit_transform(X=real_test[['ps_reg_03']] ).ravel()
real_test['ps_car_11'] = mode_imp.fit_transform(X=real_test[['ps_car_11']] ).ravel()
real_test['ps_car_12'] = mode_imp.fit_transform(X=real_test[['ps_car_12']] ).ravel()
real_test['ps_car_14'] = mean_imp.fit_transform(X=real_test[['ps_car_14']] ).ravel()
df_mask = df_segmentation.copy()
df_label = df_classification.copy()<feature_engineering>
cat_cols = meta[(meta.level=='nominal')& meta.keep].index
for col in cat_cols:
distinct_values = real_train[col].value_counts().shape[0]
print('Categorical column {} has {} distinct values'.format(col, distinct_values))
assert(np.all(df_mask['ImageId_ClassId'].values == df_label['ImageId_ClassId'].values))
print(( df_mask.loc[df_label['EncodedPixels']=='','EncodedPixels'] != '' ).sum())
 df_mask.loc[df_label['EncodedPixels']=='','EncodedPixels']=''<save_to_csv>
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def target_encode(trn_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):

 assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return add_noise(ft_trn_series, noise_level), add_noise(ft_tst_series, noise_level )
df_mask.to_csv("submission.csv", index=False )<data_type_conversions>
train_encoded, test_encoded = target_encode(train["ps_car_11_cat"],
test["ps_car_11_cat"],
target=train.target,
min_samples_leaf=100,
smoothing=10,
noise_level=0.01)
 real_train['ps_car_11_cat_te'] = train_encoded
real_train.drop('ps_car_11_cat', axis=1, inplace=True)
 meta.loc['ps_car_11_cat','keep'] = False
real_test['ps_car_11_cat_te'] = test_encoded
real_test.drop('ps_car_11_cat', axis=1, inplace=True )
if 1:
df_mask['Class'] = df_mask['ImageId_ClassId'].str[-1].astype(np.int32)
 df_mask['Label'] =(df_mask['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df_mask['Class']==1)&(df_mask['Label']==1)).sum()
pos2 =(( df_mask['Class']==2)&(df_mask['Label']==1)).sum()
pos3 =(( df_mask['Class']==3)&(df_mask['Label']==1)).sum()
pos4 =(( df_mask['Class']==4)&(df_mask['Label']==1)).sum()
num_image = len(df_mask)//4
num = len(df_mask)
 pos =(df_mask['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))<import_modules>
calc_cols = [col for col in train.columns if 'calc' in col]
real_train_nocalc = real_train.drop(calc_cols, axis=1)
 real_test_nocalc = real_test.drop(calc_cols, axis=1)
 real_train_nocalc.shape
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.torch import ToTensor
import torch.utils.data as data 
<categorify>
bin_cols = meta[(meta.level == 'binary')&(meta.role != 'target')& meta.keep].index
ones_list = []
zeros_list = []
for col in bin_cols:
zeros_nb =(real_train[col] == 0 ).sum()
ones_nb = real_train.shape[0] - zeros_nb
ones_list.append(ones_nb)
 zeros_list.append(zeros_nb)
 print('Binary column {} has {} records({:.2%})with value zero'.format(col, zeros_nb, zeros_nb/real_train.shape[0]))
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<categorify>
imbalanced_cols = ['ps_ind_10_bin', 'ps_ind_11_bin', 'ps_ind_13_bin']
real_train = real_train.drop(imbalanced_cols, axis=1)
 real_test = real_test.drop(imbalanced_cols, axis=1)
 real_train_nocalc = real_train_nocalc.drop(imbalanced_cols, axis=1)
 real_test_nocalc = real_test_nocalc.drop(imbalanced_cols, axis=1 )
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<categorify>
cat_cols = meta[(meta.level=='nominal')& meta.keep].index
print('Nb of columns in train data before dummification: {}'.format(real_train.shape[1]))
real_train = pd.get_dummies(data=real_train, columns=cat_cols, drop_first=True)
 print('Nb of columns in train data before dummification: {}'.format(real_train.shape[1]))
print('Nb of columns in test data before dummification: {}'.format(real_test.shape[1]))
real_test = pd.get_dummies(data=real_test, columns=cat_cols, drop_first=True)
 print('Nb of columns in test data after dummification: {}'.format(real_test.shape[1]))
real_train_nocalc = pd.get_dummies(data=real_train_nocalc, columns=cat_cols, drop_first=True)
 real_test_nocalc = pd.get_dummies(data=real_test_nocalc, columns=cat_cols, drop_first=True )
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num 
<define_variables>
float_cols = meta[(meta.level=='interval')& meta.keep].index
poly = PolynomialFeatures(degree=2, include_bias=False, interaction_only=False)
 interactions = pd.DataFrame(data=poly.fit_transform(real_train[float_cols]), columns=poly.get_feature_names(float_cols))
interactions.drop(float_cols, axis=1, inplace=True)
 interacted_train = pd.concat(objs=[real_train, interactions], axis=1)
 interacted_test = pd.concat(objs=[real_test, interactions], axis=1 )
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"<load_from_csv>
train_data, val_data = train_test_split(real_train, train_size=0.9, random_state=77 )
best_threshold = 0.5
num_workers = 2
batch_size = 4
print('best_threshold', best_threshold)
 min_size = 3500
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)<categorify>
class ResNetEncoder(ResNet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.fc
def forward(self, x):
x0 = self.conv1(x)
 x0 = self.bn1(x0)
 x0 = self.relu(x0)
 x1 = self.maxpool(x0)
 x1 = self.layer1(x1)
 x2 = self.layer2(x1)
 x3 = self.layer3(x2)
 x4 = self.layer4(x3)
 return [x4, x3, x2, x1, x0]
def load_state_dict(self, state_dict, **kwargs):
state_dict.pop('fc.bias')
 state_dict.pop('fc.weight')
 super().load_state_dict(state_dict, **kwargs)
 resnet_encoders = {
'resnet18': {
'encoder': ResNetEncoder,
'out_shapes':(512, 256, 128, 64, 64),
'params': {
'block': BasicBlock,
'layers': [2, 2, 2, 2],
},
},
'resnet34': {
'encoder': ResNetEncoder,
'out_shapes':(512, 256, 128, 64, 64),
'params': {
'block': BasicBlock,
'layers': [3, 4, 6, 3],
},
},
'resnet50': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 6, 3],
},
},
'resnet101': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 23, 3],
},
},
'resnet152': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 8, 36, 3],
},
},
'resnext50_32x4d': {
'encoder': ResNetEncoder,
'pretrained_settings': {
'imagenet': {
'url': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 6, 3],
'groups': 32,
'width_per_group': 4
},
},
'resnext101_32x8d': {
'encoder': ResNetEncoder,
'pretrained_settings': {
'imagenet': {
'url': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
},
'instagram': {
'url': 'https://download.pytorch.org/models/ig_resnext101_32x8-c38310e5.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 23, 3],
'groups': 32,
'width_per_group': 8
},
},
'resnext101_32x16d': {
'encoder': ResNetEncoder,
'pretrained_settings': {
'instagram':{
'url': 'https://download.pytorch.org/models/ig_resnext101_32x16-c6f796b0.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 23, 3],
'groups': 32,
'width_per_group': 16
},
},
'resnext101_32x32d': {
'encoder': ResNetEncoder,
'pretrained_settings': {
'instagram': {
'url': 'https://download.pytorch.org/models/ig_resnext101_32x32-e4b90b00.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 23, 3],
'groups': 32,
'width_per_group': 32
},
},
'resnext101_32x48d': {
'encoder': ResNetEncoder,
'pretrained_settings': {
'instagram': {
'url': 'https://download.pytorch.org/models/ig_resnext101_32x48-3e41cc8a.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 23, 3],
'groups': 32,
'width_per_group': 48
},
},
}<define_search_space>
__all__ = ['DPN', 'dpn68', 'dpn68b', 'dpn92', 'dpn98', 'dpn131', 'dpn107']
pretrained_settings = {
'dpn68': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/dpn68-4af7d88d2.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [124 / 255, 117 / 255, 104 / 255],
'std': [1 / (.0167 * 255)] * 3,
'num_classes': 1000
}
},
'dpn68b': {
'imagenet+5k': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/dpn68b_extra-363ab9c19.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [124 / 255, 117 / 255, 104 / 255],
'std': [1 / (.0167 * 255)] * 3,
'num_classes': 1000
}
},
'dpn92': {
'imagenet+5k': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/dpn92_extra-fda993c95.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [124 / 255, 117 / 255, 104 / 255],
'std': [1 / (.0167 * 255)] * 3,
'num_classes': 1000
}
},
'dpn98': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/dpn98-722954780.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [124 / 255, 117 / 255, 104 / 255],
'std': [1 / (.0167 * 255)] * 3,
'num_classes': 1000
}
},
'dpn131': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/dpn131-7af84be88.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [124 / 255, 117 / 255, 104 / 255],
'std': [1 / (.0167 * 255)] * 3,
'num_classes': 1000
}
},
'dpn107': {
'imagenet+5k': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/dpn107_extra-b7f9f4cc9.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [124 / 255, 117 / 255, 104 / 255],
'std': [1 / (.0167 * 255)] * 3,
'num_classes': 1000
}
}
}
def dpn68(num_classes=1000, pretrained='imagenet'):
model = DPN(
 small=True, num_init_features=10, k_r=128, groups=32,
k_sec=(3, 4, 12, 3), inc_sec=(16, 32, 32, 64),
num_classes=num_classes, test_time_pool=True)
 if pretrained:
settings = pretrained_settings['dpn68'][pretrained]
assert num_classes == settings['num_classes'], \
"num_classes should be {}, but is {}".format(settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
return model
def dpn68b(num_classes=1000, pretrained='imagenet+5k'):
model = DPN(
 small=True, num_init_features=10, k_r=128, groups=32,
b=True, k_sec=(3, 4, 12, 3), inc_sec=(16, 32, 32, 64),
num_classes=num_classes, test_time_pool=True)
 if pretrained:
settings = pretrained_settings['dpn68b'][pretrained]
assert num_classes == settings['num_classes'], \
"num_classes should be {}, but is {}".format(settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
return model
def dpn92(num_classes=1000, pretrained='imagenet+5k'):
model = DPN(
 num_init_features=64, k_r=96, groups=32,
k_sec=(3, 4, 20, 3), inc_sec=(16, 32, 24, 128),
num_classes=num_classes, test_time_pool=True)
 if pretrained:
settings = pretrained_settings['dpn92'][pretrained]
assert num_classes == settings['num_classes'], \
"num_classes should be {}, but is {}".format(settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
return model
def dpn98(num_classes=1000, pretrained='imagenet'):
model = DPN(
 num_init_features=96, k_r=160, groups=40,
k_sec=(3, 6, 20, 3), inc_sec=(16, 32, 32, 128),
num_classes=num_classes, test_time_pool=True)
 if pretrained:
settings = pretrained_settings['dpn98'][pretrained]
assert num_classes == settings['num_classes'], \
"num_classes should be {}, but is {}".format(settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
return model
def dpn131(num_classes=1000, pretrained='imagenet'):
model = DPN(
 num_init_features=128, k_r=160, groups=40,
k_sec=(4, 8, 28, 3), inc_sec=(16, 32, 32, 128),
num_classes=num_classes, test_time_pool=True)
 if pretrained:
settings = pretrained_settings['dpn131'][pretrained]
assert num_classes == settings['num_classes'], \
"num_classes should be {}, but is {}".format(settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
return model
def dpn107(num_classes=1000, pretrained='imagenet+5k'):
model = DPN(
 num_init_features=128, k_r=200, groups=50,
k_sec=(4, 8, 20, 3), inc_sec=(20, 64, 64, 128),
num_classes=num_classes, test_time_pool=True)
 if pretrained:
settings = pretrained_settings['dpn107'][pretrained]
assert num_classes == settings['num_classes'], \
"num_classes should be {}, but is {}".format(settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
return model
class CatBnAct(nn.Module):
def __init__(self, in_chs, activation_fn=nn.ReLU(inplace=True)) :
super(CatBnAct, self ).__init__()
self.bn = nn.BatchNorm2d(in_chs, eps=0.001)
 self.act = activation_fn
def forward(self, x):
x = torch.cat(x, dim=1)if isinstance(x, tuple)else x
return self.act(self.bn(x))
class BnActConv2d(nn.Module):
def __init__(self, in_chs, out_chs, kernel_size, stride,
padding=0, groups=1, activation_fn=nn.ReLU(inplace=True)) :
super(BnActConv2d, self ).__init__()
self.bn = nn.BatchNorm2d(in_chs, eps=0.001)
 self.act = activation_fn
self.conv = nn.Conv2d(in_chs, out_chs, kernel_size, stride, padding, groups=groups, bias=False)
 def forward(self, x):
return self.conv(self.act(self.bn(x)))
 class InputBlock(nn.Module):
def __init__(self, num_init_features, kernel_size=7,
padding=3, activation_fn=nn.ReLU(inplace=True)) :
super(InputBlock, self ).__init__()
self.conv = nn.Conv2d(
 3, num_init_features, kernel_size=kernel_size, stride=2, padding=padding, bias=False)
 self.bn = nn.BatchNorm2d(num_init_features, eps=0.001)
 self.act = activation_fn
self.pool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 def forward(self, x):
x = self.conv(x)
 x = self.bn(x)
 x = self.act(x)
 x = self.pool(x)
 return x
class DualPathBlock(nn.Module):
def __init__(
 self, in_chs, num_1x1_a, num_3x3_b, num_1x1_c, inc, groups, block_type='normal', b=False):
super(DualPathBlock, self ).__init__()
self.num_1x1_c = num_1x1_c
self.inc = inc
self.b = b
if block_type is 'proj':
self.key_stride = 1
self.has_proj = True
elif block_type is 'down':
self.key_stride = 2
self.has_proj = True
else:
assert block_type is 'normal'
self.key_stride = 1
self.has_proj = False
if self.has_proj:
if self.key_stride == 2:
self.c1x1_w_s2 = BnActConv2d(
 in_chs=in_chs, out_chs=num_1x1_c + 2 * inc, kernel_size=1, stride=2)
 else:
self.c1x1_w_s1 = BnActConv2d(
 in_chs=in_chs, out_chs=num_1x1_c + 2 * inc, kernel_size=1, stride=1)
 self.c1x1_a = BnActConv2d(in_chs=in_chs, out_chs=num_1x1_a, kernel_size=1, stride=1)
 self.c3x3_b = BnActConv2d(
 in_chs=num_1x1_a, out_chs=num_3x3_b, kernel_size=3,
stride=self.key_stride, padding=1, groups=groups)
 if b:
self.c1x1_c = CatBnAct(in_chs=num_3x3_b)
 self.c1x1_c1 = nn.Conv2d(num_3x3_b, num_1x1_c, kernel_size=1, bias=False)
 self.c1x1_c2 = nn.Conv2d(num_3x3_b, inc, kernel_size=1, bias=False)
 else:
self.c1x1_c = BnActConv2d(in_chs=num_3x3_b, out_chs=num_1x1_c + inc, kernel_size=1, stride=1)
 def forward(self, x):
x_in = torch.cat(x, dim=1)if isinstance(x, tuple)else x
if self.has_proj:
if self.key_stride == 2:
x_s = self.c1x1_w_s2(x_in)
 else:
x_s = self.c1x1_w_s1(x_in)
 x_s1 = x_s[:, :self.num_1x1_c, :, :]
x_s2 = x_s[:, self.num_1x1_c:, :, :]
else:
x_s1 = x[0]
x_s2 = x[1]
x_in = self.c1x1_a(x_in)
 x_in = self.c3x3_b(x_in)
 if self.b:
x_in = self.c1x1_c(x_in)
 out1 = self.c1x1_c1(x_in)
 out2 = self.c1x1_c2(x_in)
 else:
x_in = self.c1x1_c(x_in)
 out1 = x_in[:, :self.num_1x1_c, :, :]
out2 = x_in[:, self.num_1x1_c:, :, :]
resid = x_s1 + out1
dense = torch.cat([x_s2, out2], dim=1)
 return resid, dense
class DPN(nn.Module):
def __init__(self, small=False, num_init_features=64, k_r=96, groups=32,
b=False, k_sec=(3, 4, 20, 3), inc_sec=(16, 32, 24, 128),
num_classes=1000, test_time_pool=False):
super(DPN, self ).__init__()
self.test_time_pool = test_time_pool
self.b = b
bw_factor = 1 if small else 4
blocks = OrderedDict()
if small:
blocks['conv1_1'] = InputBlock(num_init_features, kernel_size=3, padding=1)
 else:
blocks['conv1_1'] = InputBlock(num_init_features, kernel_size=7, padding=3)
 bw = 64 * bw_factor
inc = inc_sec[0]
r =(k_r * bw)//(64 * bw_factor)
 blocks['conv2_1'] = DualPathBlock(num_init_features, r, r, bw, inc, groups, 'proj', b)
 in_chs = bw + 3 * inc
for i in range(2, k_sec[0] + 1):
blocks['conv2_' + str(i)] = DualPathBlock(in_chs, r, r, bw, inc, groups, 'normal', b)
 in_chs += inc
bw = 128 * bw_factor
inc = inc_sec[1]
r =(k_r * bw)//(64 * bw_factor)
 blocks['conv3_1'] = DualPathBlock(in_chs, r, r, bw, inc, groups, 'down', b)
 in_chs = bw + 3 * inc
for i in range(2, k_sec[1] + 1):
blocks['conv3_' + str(i)] = DualPathBlock(in_chs, r, r, bw, inc, groups, 'normal', b)
 in_chs += inc
bw = 256 * bw_factor
inc = inc_sec[2]
r =(k_r * bw)//(64 * bw_factor)
 blocks['conv4_1'] = DualPathBlock(in_chs, r, r, bw, inc, groups, 'down', b)
 in_chs = bw + 3 * inc
for i in range(2, k_sec[2] + 1):
blocks['conv4_' + str(i)] = DualPathBlock(in_chs, r, r, bw, inc, groups, 'normal', b)
 in_chs += inc
bw = 512 * bw_factor
inc = inc_sec[3]
r =(k_r * bw)//(64 * bw_factor)
 blocks['conv5_1'] = DualPathBlock(in_chs, r, r, bw, inc, groups, 'down', b)
 in_chs = bw + 3 * inc
for i in range(2, k_sec[3] + 1):
blocks['conv5_' + str(i)] = DualPathBlock(in_chs, r, r, bw, inc, groups, 'normal', b)
 in_chs += inc
blocks['conv5_bn_ac'] = CatBnAct(in_chs)
 self.features = nn.Sequential(blocks)
 self.last_linear = nn.Conv2d(in_chs, num_classes, kernel_size=1, bias=True)
 def logits(self, features):
if not self.training and self.test_time_pool:
x = F.avg_pool2d(features, kernel_size=7, stride=1)
 out = self.last_linear(x)
 out = adaptive_avgmax_pool2d(out, pool_type='avgmax')
 else:
x = adaptive_avgmax_pool2d(features, pool_type='avg')
 out = self.last_linear(x)
 return out.view(out.size(0), -1)
 def forward(self, input):
x = self.features(input)
 x = self.logits(x)
 return x

 def pooling_factor(pool_type='avg'):
return 2 if pool_type == 'avgmaxc' else 1
def adaptive_avgmax_pool2d(x, pool_type='avg', padding=0, count_include_pad=False):

 if pool_type == 'avgmaxc':
x = torch.cat([
F.avg_pool2d(
 x, kernel_size=(x.size(2), x.size(3)) , padding=padding, count_include_pad=count_include_pad),
F.max_pool2d(x, kernel_size=(x.size(2), x.size(3)) , padding=padding)
 ], dim=1)
 elif pool_type == 'avgmax':
x_avg = F.avg_pool2d(
 x, kernel_size=(x.size(2), x.size(3)) , padding=padding, count_include_pad=count_include_pad)
 x_max = F.max_pool2d(x, kernel_size=(x.size(2), x.size(3)) , padding=padding)
 x = 0.5 *(x_avg + x_max)
 elif pool_type == 'max':
x = F.max_pool2d(x, kernel_size=(x.size(2), x.size(3)) , padding=padding)
 else:
if pool_type != 'avg':
print('Invalid pool type %s specified.Defaulting to average pooling.' % pool_type)
 x = F.avg_pool2d(
 x, kernel_size=(x.size(2), x.size(3)) , padding=padding, count_include_pad=count_include_pad)
 return x
class AdaptiveAvgMaxPool2d(torch.nn.Module):

 def __init__(self, output_size=1, pool_type='avg'):
super(AdaptiveAvgMaxPool2d, self ).__init__()
self.output_size = output_size
self.pool_type = pool_type
if pool_type == 'avgmaxc' or pool_type == 'avgmax':
self.pool = nn.ModuleList([nn.AdaptiveAvgPool2d(output_size), nn.AdaptiveMaxPool2d(output_size)])
 elif pool_type == 'max':
self.pool = nn.AdaptiveMaxPool2d(output_size)
 else:
if pool_type != 'avg':
print('Invalid pool type %s specified.Defaulting to average pooling.' % pool_type)
 self.pool = nn.AdaptiveAvgPool2d(output_size)
 def forward(self, x):
if self.pool_type == 'avgmaxc':
x = torch.cat([p(x)for p in self.pool], dim=1)
 elif self.pool_type == 'avgmax':
x = 0.5 * torch.sum(torch.stack([p(x)for p in self.pool]), 0 ).squeeze(dim=0)
 else:
x = self.pool(x)
 return x
def factor(self):
return pooling_factor(self.pool_type)
 def __repr__(self):
return self.__class__.__name__ + '(' \
+ 'output_size=' + str(self.output_size)\
+ ', pool_type=' + self.pool_type + ')'
class DPNEncorder(DPN):
def __init__(self, feature_blocks, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.feature_blocks = np.cumsum(feature_blocks)
 self.pretrained = False
del self.last_linear
def forward(self, x):
features = []
input_block = self.features[0]
x = input_block.conv(x)
 x = input_block.bn(x)
 x = input_block.act(x)
 features.append(x)
 x = input_block.pool(x)
 for i, module in enumerate(self.features[1:], 1):
x = module(x)
 if i in self.feature_blocks:
features.append(x)
 out_features = [
features[4],
F.relu(torch.cat(features[3], dim=1), inplace=True),
F.relu(torch.cat(features[2], dim=1), inplace=True),
F.relu(torch.cat(features[1], dim=1), inplace=True),
features[0],
]
return out_features
def load_state_dict(self, state_dict, **kwargs):
state_dict.pop('last_linear.bias')
 state_dict.pop('last_linear.weight')
 super().load_state_dict(state_dict, **kwargs)
 dpn_encoders = {
'dpn68': {
'encoder': DPNEncorder,
'out_shapes':(832, 704, 320, 144, 10),
'params': {
'feature_blocks':(3, 4, 12, 4),
'groups': 32,
'inc_sec':(16, 32, 32, 64),
'k_r': 128,
'k_sec':(3, 4, 12, 3),
'num_classes': 1000,
'num_init_features': 10,
'small': True,
'test_time_pool': True
},
},
'dpn68b': {
'encoder': DPNEncorder,
'out_shapes':(832, 704, 320, 144, 10),
'params': {
'feature_blocks':(3, 4, 12, 4),
'b': True,
'groups': 32,
'inc_sec':(16, 32, 32, 64),
'k_r': 128,
'k_sec':(3, 4, 12, 3),
'num_classes': 1000,
'num_init_features': 10,
'small': True,
'test_time_pool': True,
},
},
'dpn92': {
'encoder': DPNEncorder,
'out_shapes':(2688, 1552, 704, 336, 64),
'params': {
'feature_blocks':(3, 4, 20, 4),
'groups': 32,
'inc_sec':(16, 32, 24, 128),
'k_r': 96,
'k_sec':(3, 4, 20, 3),
'num_classes': 1000,
'num_init_features': 64,
'test_time_pool': True
},
},
'dpn98': {
'encoder': DPNEncorder,
'out_shapes':(2688, 1728, 768, 336, 96),
'pretrained_settings': pretrained_settings['dpn98'],
'params': {
'feature_blocks':(3, 6, 20, 4),
'groups': 40,
'inc_sec':(16, 32, 32, 128),
'k_r': 160,
'k_sec':(3, 6, 20, 3),
'num_classes': 1000,
'num_init_features': 96,
'test_time_pool': True,
},
},
'dpn107': {
'encoder': DPNEncorder,
'out_shapes':(2688, 2432, 1152, 376, 128),
'params': {
'feature_blocks':(4, 8, 20, 4),
'groups': 50,
'inc_sec':(20, 64, 64, 128),
'k_r': 200,
'k_sec':(4, 8, 20, 3),
'num_classes': 1000,
'num_init_features': 128,
'test_time_pool': True
},
},
'dpn131': {
'encoder': DPNEncorder,
'out_shapes':(2688, 1984, 832, 352, 128),
'params': {
'feature_blocks':(4, 8, 28, 4),
'groups': 40,
'inc_sec':(16, 32, 32, 128),
'k_r': 160,
'k_sec':(4, 8, 28, 3),
'num_classes': 1000,
'num_init_features': 128,
'test_time_pool': True
},
},
}<define_search_space>
__all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',
'se_resnext50_32x4d', 'se_resnext101_32x4d']
pretrained_settings = {
'senet154': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet50': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet101': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet152': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext50_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext101_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
}
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def initialize_pretrained_model(model, num_classes, settings):
assert num_classes == settings['num_classes'], \
'num_classes should be {}, but is {}'.format(
 settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
def senet154(num_classes=1000, pretrained='imagenet'):
model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,
dropout_p=0.2, num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['senet154'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet50(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet50'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet101(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet101'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet152(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet152'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext50_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext101_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
class SENetEncoder(SENet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.last_linear
del self.avg_pool
def forward(self, x):
for module in self.layer0[:-1]:
x = module(x)
 x0 = x
x = self.layer0[-1](x)
 x1 = self.layer1(x)
 x2 = self.layer2(x1)
 x3 = self.layer3(x2)
 x4 = self.layer4(x3)
 features = [x4, x3, x2, x1, x0]
return features
def load_state_dict(self, state_dict, **kwargs):
state_dict.pop('last_linear.bias')
 state_dict.pop('last_linear.weight')
 super().load_state_dict(state_dict, **kwargs)
 senet_encoders = {
'senet154': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 128),
'params': {
'block': SEBottleneck,
'dropout_p': 0.2,
'groups': 64,
'layers': [3, 8, 36, 3],
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet50': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 4, 6, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet101': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 4, 23, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet152': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 8, 36, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnext50_32x4d': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNeXtBottleneck,
'layers': [3, 4, 6, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 32,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnext101_32x4d': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNeXtBottleneck,
'layers': [3, 4, 23, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 32,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
}<define_search_space>
def gini(actual, pred, cmpcol = 0, sortcol = 1):
assert(len(actual)== len(pred))
all = np.asarray(np.c_[ actual, pred, np.arange(len(actual)) ], dtype=np.float)
 all = all[ np.lexsort(( all[:,2], -1*all[:,1])) ]
totalLosses = all[:,0].sum()
giniSum = all[:,0].cumsum().sum() / totalLosses
giniSum -=(len(actual)+ 1)/ 2.
 return giniSum / len(actual)
 def gini_normalized(a, p):
return gini(a, p)/ gini(a, a)
 def gini_xgb(preds, d_train):
targets = d_train.get_label()
gini_score = gini_normalized(targets, preds)
 return [('gini', gini_score)]
__all__ = ['InceptionResNetV2', 'inceptionresnetv2']
pretrained_settings = {
'inceptionresnetv2': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionresnetv2-520b38e4.pth',
'input_space': 'RGB',
'input_size': [3, 299, 299],
'input_range': [0, 1],
'mean': [0.5, 0.5, 0.5],
'std': [0.5, 0.5, 0.5],
'num_classes': 1000
},
'imagenet+background': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionresnetv2-520b38e4.pth',
'input_space': 'RGB',
'input_size': [3, 299, 299],
'input_range': [0, 1],
'mean': [0.5, 0.5, 0.5],
'std': [0.5, 0.5, 0.5],
'num_classes': 1001
}
}
}
class BasicConv2d(nn.Module):
def __init__(self, in_planes, out_planes, kernel_size, stride, padding=0):
super(BasicConv2d, self ).__init__()
self.conv = nn.Conv2d(in_planes, out_planes,
kernel_size=kernel_size, stride=stride,
padding=padding, bias=False)
 self.bn = nn.BatchNorm2d(out_planes,
eps=0.001,
momentum=0.1,
affine=True)
 self.relu = nn.ReLU(inplace=False)
 def forward(self, x):
x = self.conv(x)
 x = self.bn(x)
 x = self.relu(x)
 return x
class Mixed_5b(nn.Module):
def __init__(self):
super(Mixed_5b, self ).__init__()
self.branch0 = BasicConv2d(192, 96, kernel_size=1, stride=1)
 self.branch1 = nn.Sequential(
 BasicConv2d(192, 48, kernel_size=1, stride=1),
BasicConv2d(48, 64, kernel_size=5, stride=1, padding=2)
)
 self.branch2 = nn.Sequential(
 BasicConv2d(192, 64, kernel_size=1, stride=1),
BasicConv2d(64, 96, kernel_size=3, stride=1, padding=1),
BasicConv2d(96, 96, kernel_size=3, stride=1, padding=1)
)
 self.branch3 = nn.Sequential(
 nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),
BasicConv2d(192, 64, kernel_size=1, stride=1)
)
 def forward(self, x):
x0 = self.branch0(x)
 x1 = self.branch1(x)
 x2 = self.branch2(x)
 x3 = self.branch3(x)
 out = torch.cat(( x0, x1, x2, x3), 1)
 return out
class Block35(nn.Module):
def __init__(self, scale=1.0):
super(Block35, self ).__init__()
self.scale = scale
self.branch0 = BasicConv2d(320, 32, kernel_size=1, stride=1)
 self.branch1 = nn.Sequential(
 BasicConv2d(320, 32, kernel_size=1, stride=1),
BasicConv2d(32, 32, kernel_size=3, stride=1, padding=1)
)
 self.branch2 = nn.Sequential(
 BasicConv2d(320, 32, kernel_size=1, stride=1),
BasicConv2d(32, 48, kernel_size=3, stride=1, padding=1),
BasicConv2d(48, 64, kernel_size=3, stride=1, padding=1)
)
 self.conv2d = nn.Conv2d(128, 320, kernel_size=1, stride=1)
 self.relu = nn.ReLU(inplace=False)
 def forward(self, x):
x0 = self.branch0(x)
 x1 = self.branch1(x)
 x2 = self.branch2(x)
 out = torch.cat(( x0, x1, x2), 1)
 out = self.conv2d(out)
 out = out * self.scale + x
out = self.relu(out)
 return out
class Mixed_6a(nn.Module):
def __init__(self):
super(Mixed_6a, self ).__init__()
self.branch0 = BasicConv2d(320, 384, kernel_size=3, stride=2)
 self.branch1 = nn.Sequential(
 BasicConv2d(320, 256, kernel_size=1, stride=1),
BasicConv2d(256, 256, kernel_size=3, stride=1, padding=1),
BasicConv2d(256, 384, kernel_size=3, stride=2)
)
 self.branch2 = nn.MaxPool2d(3, stride=2)
 def forward(self, x):
x0 = self.branch0(x)
 x1 = self.branch1(x)
 x2 = self.branch2(x)
 out = torch.cat(( x0, x1, x2), 1)
 return out
class Block17(nn.Module):
def __init__(self, scale=1.0):
super(Block17, self ).__init__()
self.scale = scale
self.branch0 = BasicConv2d(1088, 192, kernel_size=1, stride=1)
 self.branch1 = nn.Sequential(
 BasicConv2d(1088, 128, kernel_size=1, stride=1),
BasicConv2d(128, 160, kernel_size=(1,7), stride=1, padding=(0,3)) ,
BasicConv2d(160, 192, kernel_size=(7,1), stride=1, padding=(3,0)) 
)
 self.conv2d = nn.Conv2d(384, 1088, kernel_size=1, stride=1)
 self.relu = nn.ReLU(inplace=False)
 def forward(self, x):
x0 = self.branch0(x)
 x1 = self.branch1(x)
 out = torch.cat(( x0, x1), 1)
 out = self.conv2d(out)
 out = out * self.scale + x
out = self.relu(out)
 return out
class Mixed_7a(nn.Module):
def __init__(self):
super(Mixed_7a, self ).__init__()
self.branch0 = nn.Sequential(
 BasicConv2d(1088, 256, kernel_size=1, stride=1),
BasicConv2d(256, 384, kernel_size=3, stride=2)
)
 self.branch1 = nn.Sequential(
 BasicConv2d(1088, 256, kernel_size=1, stride=1),
BasicConv2d(256, 288, kernel_size=3, stride=2)
)
 self.branch2 = nn.Sequential(
 BasicConv2d(1088, 256, kernel_size=1, stride=1),
BasicConv2d(256, 288, kernel_size=3, stride=1, padding=1),
BasicConv2d(288, 320, kernel_size=3, stride=2)
)
 self.branch3 = nn.MaxPool2d(3, stride=2)
 def forward(self, x):
x0 = self.branch0(x)
 x1 = self.branch1(x)
 x2 = self.branch2(x)
 x3 = self.branch3(x)
 out = torch.cat(( x0, x1, x2, x3), 1)
 return out
class Block8(nn.Module):
def __init__(self, scale=1.0, noReLU=False):
super(Block8, self ).__init__()
self.scale = scale
self.noReLU = noReLU
self.branch0 = BasicConv2d(2080, 192, kernel_size=1, stride=1)
 self.branch1 = nn.Sequential(
 BasicConv2d(2080, 192, kernel_size=1, stride=1),
BasicConv2d(192, 224, kernel_size=(1,3), stride=1, padding=(0,1)) ,
BasicConv2d(224, 256, kernel_size=(3,1), stride=1, padding=(1,0)) 
)
 self.conv2d = nn.Conv2d(448, 2080, kernel_size=1, stride=1)
 if not self.noReLU:
self.relu = nn.ReLU(inplace=False)
 def forward(self, x):
x0 = self.branch0(x)
 x1 = self.branch1(x)
 out = torch.cat(( x0, x1), 1)
 out = self.conv2d(out)
 out = out * self.scale + x
if not self.noReLU:
out = self.relu(out)
 return out
class InceptionResNetV2(nn.Module):
def __init__(self, num_classes=1001):
super(InceptionResNetV2, self ).__init__()
self.input_space = None
self.input_size =(299, 299, 3)
 self.mean = None
self.std = None
self.conv2d_1a = BasicConv2d(3, 32, kernel_size=3, stride=2)
 self.conv2d_2a = BasicConv2d(32, 32, kernel_size=3, stride=1)
 self.conv2d_2b = BasicConv2d(32, 64, kernel_size=3, stride=1, padding=1)
 self.maxpool_3a = nn.MaxPool2d(3, stride=2)
 self.conv2d_3b = BasicConv2d(64, 80, kernel_size=1, stride=1)
 self.conv2d_4a = BasicConv2d(80, 192, kernel_size=3, stride=1)
 self.maxpool_5a = nn.MaxPool2d(3, stride=2)
 self.mixed_5b = Mixed_5b()
self.repeat = nn.Sequential(
 Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17)
)
 self.mixed_6a = Mixed_6a()
self.repeat_1 = nn.Sequential(
 Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10)
)
 self.mixed_7a = Mixed_7a()
self.repeat_2 = nn.Sequential(
 Block8(scale=0.20),
Block8(scale=0.20),
Block8(scale=0.20),
Block8(scale=0.20),
Block8(scale=0.20),
Block8(scale=0.20),
Block8(scale=0.20),
Block8(scale=0.20),
Block8(scale=0.20)
)
 self.block8 = Block8(noReLU=True)
 self.conv2d_7b = BasicConv2d(2080, 1536, kernel_size=1, stride=1)
 self.avgpool_1a = nn.AvgPool2d(8, count_include_pad=False)
 self.last_linear = nn.Linear(1536, num_classes)
 def features(self, input):
x = self.conv2d_1a(input)
 x = self.conv2d_2a(x)
 x = self.conv2d_2b(x)
 x = self.maxpool_3a(x)
 x = self.conv2d_3b(x)
 x = self.conv2d_4a(x)
 x = self.maxpool_5a(x)
 x = self.mixed_5b(x)
 x = self.repeat(x)
 x = self.mixed_6a(x)
 x = self.repeat_1(x)
 x = self.mixed_7a(x)
 x = self.repeat_2(x)
 x = self.block8(x)
 x = self.conv2d_7b(x)
 return x
def logits(self, features):
x = self.avgpool_1a(features)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, input):
x = self.features(input)
 x = self.logits(x)
 return x
def inceptionresnetv2(num_classes=1000, pretrained='imagenet'):
r
if pretrained:
settings = pretrained_settings['inceptionresnetv2'][pretrained]
assert num_classes == settings['num_classes'], \
"num_classes should be {}, but is {}".format(settings['num_classes'], num_classes)
 model = InceptionResNetV2(num_classes=1001)
 model.load_state_dict(model_zoo.load_url(settings['url']))
if pretrained == 'imagenet':
new_last_linear = nn.Linear(1536, 1000)
 new_last_linear.weight.data = model.last_linear.weight.data[1:]
new_last_linear.bias.data = model.last_linear.bias.data[1:]
model.last_linear = new_last_linear
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
else:
model = InceptionResNetV2(num_classes=num_classes)
 return model
class InceptionResNetV2Encoder(InceptionResNetV2):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
for m in self.modules() :
if isinstance(m, nn.Conv2d):
if m.kernel_size ==(3, 3):
m.padding =(1, 1)
 if isinstance(m, nn.MaxPool2d):
m.padding =(1, 1)
 del self.avgpool_1a
del self.last_linear
def forward(self, x):
x = self.conv2d_1a(x)
 x = self.conv2d_2a(x)
 x = self.conv2d_2b(x)
 x0 = x
x = self.maxpool_3a(x)
 x = self.conv2d_3b(x)
 x = self.conv2d_4a(x)
 x1 = x
x = self.maxpool_5a(x)
 x = self.mixed_5b(x)
 x = self.repeat(x)
 x2 = x
x = self.mixed_6a(x)
 x = self.repeat_1(x)
 x3 = x
x = self.mixed_7a(x)
 x = self.repeat_2(x)
 x = self.block8(x)
 x = self.conv2d_7b(x)
 x4 = x
features = [x4, x3, x2, x1, x0]
return features
def load_state_dict(self, state_dict, **kwargs):
state_dict.pop('last_linear.bias')
 state_dict.pop('last_linear.weight')
 super().load_state_dict(state_dict, **kwargs)
 inception_encoders = {
'inceptionresnetv2': {
'encoder': InceptionResNetV2Encoder,
'out_shapes':(1536, 1088, 320, 192, 64),
'params': {
'num_classes': 1000,
}
}
}<define_variables>
d_train = xgb.DMatrix(real_train.drop(['id', 'target'], axis=1), real_train[['target']])
 d_test = xgb.DMatrix(real_test.drop(['id'], axis=1))
n_splits = 5
n_estimators = 7
folds = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=77)
 params = {
'objective': 'binary:logistic',
'eta': 0.015,
'eval_metric': 'auc',
'max_depth': 6,
'min_child_weight': 10,
'gamma': 1,
'reg_lambda': 0.3,
'reg_alpha': 0.07,
'subsample': 0.8,
'colsample_bytree': 0.8,
'silent': False,
}
xgbCV = xgboost.cv(params=params, dtrain=d_train, stratified=True, num_boost_round=10000, feval=gini_xgb,
early_stopping_rounds=100, maximize=True, verbose_eval=5, nfold=3 )
__all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',
'se_resnext50_32x4d', 'se_resnext101_32x4d']
pretrained_settings = {
'senet154': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet50': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet101': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet152': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext50_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext101_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
}
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def initialize_pretrained_model(model, num_classes, settings):
assert num_classes == settings['num_classes'], \
'num_classes should be {}, but is {}'.format(
 settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
def senet154(num_classes=1000, pretrained='imagenet'):
model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,
dropout_p=0.2, num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['senet154'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet50(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet50'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet101(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet101'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet152(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet152'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext50_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext101_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
class SENetEncoder(SENet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.last_linear
del self.avg_pool
def forward(self, x):
for module in self.layer0[:-1]:
x = module(x)
 x0 = x
x = self.layer0[-1](x)
 x1 = self.layer1(x)
 x2 = self.layer2(x1)
 x3 = self.layer3(x2)
 x4 = self.layer4(x3)
 features = [x4, x3, x2, x1, x0]
return features
def load_state_dict(self, state_dict, **kwargs):
state_dict.pop('last_linear.bias')
 state_dict.pop('last_linear.weight')
 super().load_state_dict(state_dict, **kwargs)
 senet_encoders = {
'senet154': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 128),
'params': {
'block': SEBottleneck,
'dropout_p': 0.2,
'groups': 64,
'layers': [3, 8, 36, 3],
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet50': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 4, 6, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet101': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 4, 23, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet152': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 8, 36, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnext50_32x4d': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNeXtBottleneck,
'layers': [3, 4, 6, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 32,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnext101_32x4d': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNeXtBottleneck,
'layers': [3, 4, 23, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 32,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
}
class ResNetEncoder(ResNet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.fc
def forward(self, x):
x0 = self.conv1(x)
 x0 = self.bn1(x0)
 x0 = self.relu(x0)
 x1 = self.maxpool(x0)
 x1 = self.layer1(x1)
 x2 = self.layer2(x1)
 x3 = self.layer3(x2)
 x4 = self.layer4(x3)
 return [x4, x3, x2, x1, x0]
def load_state_dict(self, state_dict, **kwargs):
state_dict.pop('fc.bias')
 state_dict.pop('fc.weight')
 super().load_state_dict(state_dict, **kwargs)
 resnet_encoders = {
'resnet18': {
'encoder': ResNetEncoder,
'out_shapes':(512, 256, 128, 64, 64),
'params': {
'block': BasicBlock,
'layers': [2, 2, 2, 2],
},
},
'resnet34': {
'encoder': ResNetEncoder,
'out_shapes':(512, 256, 128, 64, 64),
'params': {
'block': BasicBlock,
'layers': [3, 4, 6, 3],
},
},
'resnet50': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 6, 3],
},
},
'resnet101': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 23, 3],
},
},
'resnet152': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 8, 36, 3],
},
},
}
class DenseNetEncoder(DenseNet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.classifier
self.initialize()
@staticmethod
def _transition(x, transition_block):
for module in transition_block:
x = module(x)
 if isinstance(module, nn.ReLU):
skip = x
return x, skip
def initialize(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 def forward(self, x):
x = self.features.conv0(x)
 x = self.features.norm0(x)
 x = self.features.relu0(x)
 x0 = x
x = self.features.pool0(x)
 x = self.features.denseblock1(x)
 x, x1 = self._transition(x, self.features.transition1)
 x = self.features.denseblock2(x)
 x, x2 = self._transition(x, self.features.transition2)
 x = self.features.denseblock3(x)
 x, x3 = self._transition(x, self.features.transition3)
 x = self.features.denseblock4(x)
 x4 = self.features.norm5(x)
 features = [x4, x3, x2, x1, x0]
return features
def load_state_dict(self, state_dict):
pattern = re.compile(
 r'^ (.*denselayer\d+\.( ?:norm|relu|conv)) \.((?:[12])\.( ?:weight|bias|running_mean|running_var)) $')
 for key in list(state_dict.keys()):
res = pattern.match(key)
 if res:
new_key = res.group(1)+ res.group(2)
 state_dict[new_key] = state_dict[key]
del state_dict[key]
state_dict.pop('classifier.bias')
 state_dict.pop('classifier.weight')
 super().load_state_dict(state_dict)
 densenet_encoders = {
'densenet121': {
'encoder': DenseNetEncoder,
'out_shapes':(1024, 1024, 512, 256, 64),
'params': {
'num_init_features': 64,
'growth_rate': 32,
'block_config':(6, 12, 24, 16),
}
},
'densenet169': {
'encoder': DenseNetEncoder,
'out_shapes':(1664, 1280, 512, 256, 64),
'params': {
'num_init_features': 64,
'growth_rate': 32,
'block_config':(6, 12, 32, 32),
}
},
'densenet201': {
'encoder': DenseNetEncoder,
'out_shapes':(1920, 1792, 512, 256, 64),
'params': {
'num_init_features': 64,
'growth_rate': 32,
'block_config':(6, 12, 48, 32),
}
},
'densenet161': {
'encoder': DenseNetEncoder,
'out_shapes':(2208, 2112, 768, 384, 96),
'params': {
'num_init_features': 96,
'growth_rate': 48,
'block_config':(6, 12, 36, 24),
}
},
}
encoders = {}
encoders.update(resnet_encoders)
 encoders.update(senet_encoders)
 encoders.update(densenet_encoders)
 encoders.update(inception_encoders)
 def get_encoder(name, encoder_weights=None):
Encoder = encoders[name]['encoder']
encoder = Encoder(**encoders[name]['params'])
 encoder.out_shapes = encoders[name]['out_shapes']
if encoder_weights is not None:
settings = encoders[name]['pretrained_settings'][encoder_weights]
encoder.load_state_dict(model_zoo.load_url(settings['url']))
return encoder
def get_encoder_names() :
return list(encoders.keys())
 def get_preprocessing_params(encoder_name, pretrained='imagenet'):
settings = encoders[encoder_name]['pretrained_settings']
if pretrained not in settings.keys() :
raise ValueError('Avaliable pretrained options {}'.format(settings.keys()))
formatted_settings = {}
formatted_settings['input_space'] = settings[pretrained].get('input_space')
 formatted_settings['input_range'] = settings[pretrained].get('input_range')
 formatted_settings['mean'] = settings[pretrained].get('mean')
 formatted_settings['std'] = settings[pretrained].get('std')
 return formatted_settings
def get_preprocessing_fn(encoder_name, pretrained='imagenet'):
params = get_preprocessing_params(encoder_name, pretrained=pretrained)
 return functools.partial(preprocess_input, **params)
 class Model(nn.Module):
def __init__(self):
super().__init__()
def initialize(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 class EncoderDecoder(Model):
def __init__(self, encoder, decoder, activation):
super().__init__()
self.encoder = encoder
self.decoder = decoder
if callable(activation)or activation is None:
self.activation = activation
elif activation == 'softmax':
self.activation = nn.Softmax(dim=1)
 elif activation == 'sigmoid':
self.activation = nn.Sigmoid()
else:
raise ValueError('Activation should be "sigmoid"/"softmax"/callable/None')
 def forward(self, x):

 x = self.encoder(x)
 x = self.decoder(x)
 return x
def predict(self, x):

 if self.training:
self.eval()
with torch.no_grad() :
x = self.forward(x)
 if self.activation:
x = self.activation(x)
 return x<choose_model_class>
@jit
def eval_gini(y_true, y_prob):

 y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = eval_gini(labels, preds)
 return [('gini', gini_score)]
trn_df = real_train
sub_df = real_test
target = trn_df.target
del trn_df["target"]
n_splits = 5
n_estimators = 7
folds = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=77)
 imp_df = np.zeros(( len(trn_df.columns), n_splits))
xgb_evals = np.zeros(( n_estimators, n_splits))
oof = np.empty(len(trn_df))
sub_preds = np.zeros(len(sub_df))
increase = True
np.random.seed(0)
 print('Start fitting: ')
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(target, target)) :
trn_dat, trn_tgt = trn_df.iloc[trn_idx], target.iloc[trn_idx]
val_dat, val_tgt = trn_df.iloc[val_idx], target.iloc[val_idx]
clf = XGBClassifier(n_estimators=n_estimators,
max_depth=4,
objective="binary:logistic",
learning_rate=0.1,
subsample=.8,
colsample_bytree=.8,
gamma=1,
reg_alpha=0,
reg_lambda=1,
nthread=2)
 if increase:
pos = pd.Series(trn_tgt == 1)
 trn_dat = pd.concat([trn_dat, trn_dat.loc[pos]], axis=0)
 trn_tgt = pd.concat([trn_tgt, trn_tgt.loc[pos]], axis=0)
 idx = np.arange(len(trn_dat))
np.random.shuffle(idx)
 trn_dat = trn_dat.iloc[idx]
trn_tgt = trn_tgt.iloc[idx]
print('{}th fold'.format(fold_))
clf.fit(trn_dat, trn_tgt,
eval_set=[(trn_dat, trn_tgt),(val_dat, val_tgt)],
eval_metric=gini_xgb,
early_stopping_rounds=None,
verbose=False)
 imp_df[:, fold_] = clf.feature_importances_
oof[val_idx] = clf.predict_proba(val_dat)[:, 1]
xgb_evals[:, fold_] = clf.evals_result_["validation_1"]["gini"]
best_round = np.argsort(xgb_evals[:, fold_])[::-1][0]
print("Fold %2d : %.6f @%4d / best score is %.6f @%4d"
%(fold_ + 1,
eval_gini(val_tgt, oof[val_idx]),
n_estimators,
xgb_evals[best_round, fold_],
best_round))
sub_preds += clf.predict_proba(sub_df)[:, 1] / n_splits
print("Full OOF score : %.6f" % eval_gini(target, oof))
mean_eval = np.mean(xgb_evals, axis=1)
 std_eval = np.std(xgb_evals, axis=1)
 best_round = np.argsort(mean_eval)[::-1][0]
print("Best mean score : %.6f + %.6f @%4d"
%(mean_eval[best_round], std_eval[best_round], best_round))
importances = imp_df.mean(axis=1)
 for i, imp in enumerate(importances):
print("%-20s : %10.4f" %(trn_df.columns[i], imp))
sub_df["target"] = sub_preds
sub_df[["target"]].to_csv("submission_20fold.csv", index=True, float_format="%.9f" )
class Conv2dReLU(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size, padding=0,
stride=1, use_batchnorm=True, **batchnorm_params):
super().__init__()
layers = [
nn.Conv2d(in_channels, out_channels, kernel_size,
stride=stride, padding=padding, bias=not(use_batchnorm)) ,
nn.ReLU(inplace=True),
]
if use_batchnorm:
layers.insert(1, nn.BatchNorm2d(out_channels, **batchnorm_params))
self.block = nn.Sequential(*layers)
 def forward(self, x):
return self.block(x)
 class SCSEModule(nn.Module):
def __init__(self, ch, re=16):
super().__init__()
self.cSE = nn.Sequential(nn.AdaptiveAvgPool2d(1),
nn.Conv2d(ch, ch//re, 1),
nn.ReLU(inplace=True),
nn.Conv2d(ch//re, ch, 1),
nn.Sigmoid() 
)
 self.sSE = nn.Sequential(nn.Conv2d(ch, ch, 1),
nn.Sigmoid())
 def forward(self, x):
return x * self.cSE(x)+ x * self.sSE(x)
 def _upsample(x, size):
return F.interpolate(x, size=size, mode='bilinear', align_corners=True)
 class PyramidStage(nn.Module):
def __init__(self, in_channels, out_channels, pool_size, use_bathcnorm=True):
super().__init__()
if pool_size == 1:
use_bathcnorm = False
self.pool = nn.Sequential(
 nn.AdaptiveAvgPool2d(output_size=(pool_size, pool_size)) ,
Conv2dReLU(in_channels, out_channels,(1, 1), use_batchnorm=use_bathcnorm)
)
 def forward(self, x):
h, w = x.size(2), x.size(3)
 x = self.pool(x)
 x = _upsample(x, size=(h, w))
return x
class PSPModule(nn.Module):
def __init__(self, in_channels, sizes=(1, 2, 3, 6), use_bathcnorm=True):
super().__init__()
self.stages = nn.ModuleList([
PyramidStage(in_channels, in_channels // len(sizes), size, use_bathcnorm=use_bathcnorm)for size in sizes
])
 def forward(self, x):
xs = [stage(x)for stage in self.stages] + [x]
x = torch.cat(xs, dim=1)
 return x
class AUXModule(nn.Module):
def __init__(self, in_features, out_features):
super().__init__()
self.linear = nn.Linear(in_features, out_features)
 def forward(self, x):
x = F.adaptive_max_pool2d(x, output_size=(1, 1))
x = x.view(-1, x.size(1))
x = self.linear(x)
 return x
class PSPDecoder(Model):
def __init__(
 self,
encoder_channels,
downsample_factor=8,
use_batchnorm=True,
psp_out_channels=512,
final_channels=21,
aux_output=False,
dropout=0.2, 
):
super().__init__()
self.downsample_factor = downsample_factor
self.out_channels = self._get(encoder_channels)
 self.aux_output = aux_output
self.dropout_factor = dropout
self.psp = PSPModule(
 self.out_channels,
sizes=(1, 2, 3, 6),
use_bathcnorm=use_batchnorm, 
)
 self.conv = Conv2dReLU(
 self.out_channels * 2,
psp_out_channels,
kernel_size=1,
use_batchnorm=use_batchnorm, 
)
 if self.dropout_factor:
self.dropout = nn.Dropout2d(p=dropout)
 self.final_conv = nn.Conv2d(psp_out_channels, final_channels,
kernel_size=(3, 3), padding=1)
 if self.aux_output:
self.aux = AUXModule(self.out_channels, final_channels)
 self.initialize()
def _get(self, xs):
if self.downsample_factor == 4:
return xs[3]
elif self.downsample_factor == 8:
return xs[2]
elif self.downsample_factor == 16:
return xs[1]
else:
raise ValueError('Downsample factor should bi in [4, 8, 16], got {}' 
.format(self.downsample_factor))
def forward(self, x):
features = self._get(x)
 x = self.psp(features)
 x = self.conv(x)
 if self.dropout_factor:
x = self.dropout(x)
 x = self.final_conv(x)
 x = F.interpolate(
 x,
scale_factor=self.downsample_factor,
mode='bilinear',
align_corners=True 
)
 if self.training and self.aux_output:
aux = self.aux(features)
 x = [x, aux]
return x
class PSPNet(EncoderDecoder):

 def __init__(
 self,
encoder_name='resnet34',
encoder_weights='imagenet',
psp_in_factor=8,
psp_out_channels=512,
psp_use_batchnorm=True,
psp_aux_output=False,
classes=21,
dropout=0.2,
activation='softmax', 
):
encoder = get_encoder(
 encoder_name,
encoder_weights=encoder_weights 
)
 decoder = PSPDecoder(
 encoder_channels=encoder.out_shapes,
downsample_factor=psp_in_factor,
psp_out_channels=psp_out_channels,
final_channels=classes,
dropout=dropout,
aux_output=psp_aux_output,
use_batchnorm=psp_use_batchnorm, 
)
 super().__init__(encoder, decoder, activation)
 self.name = 'psp-{}'.format(encoder_name )<choose_model_class>
d_train = xgb.DMatrix(train_data.drop(['id', 'target'], axis=1), train_data[['target']])
 d_val = xgb.DMatrix(val_data.drop(['id', 'target'], axis=1), val_data[['target']])
 d_test = xgb.DMatrix(real_test.drop(['id'], axis=1))
params = {
'objective': 'binary:logistic',
'eta': 0.015,
'eval_metric': 'auc',
'max_depth': 6,
'min_child_weight': 10,
'gamma': 1,
'reg_lambda': 0.3,
'reg_alpha': 0.07,
'subsample': 0.8,
'colsample_bytree': 0.8,
'silent': True,
'n_estimators': 400,
}
watchlist = [(d_train, 'train'),(d_val, 'valid')]
mdl_xgb = xgb.train(params, d_train, num_boost_round=10000, evals=watchlist, early_stopping_rounds=100,
feval=gini_xgb, maximize=True, verbose_eval=10)
 filename = 'xgb_model.joblib.pkl'
_ = joblib.dump(mdl_xgb, filename, compress=9 )
class DecoderBlock(nn.Module):
def __init__(self, in_channels, out_channels, use_batchnorm=True, attention_type=None):
super().__init__()
if attention_type is None:
self.attention1 = nn.Identity()
self.attention2 = nn.Identity()
elif attention_type == 'scse':
self.attention1 = SCSEModule(in_channels)
 self.attention2 = SCSEModule(out_channels)
 self.block = nn.Sequential(
 Conv2dReLU(in_channels, out_channels, kernel_size=3, padding=1, use_batchnorm=use_batchnorm),
Conv2dReLU(out_channels, out_channels, kernel_size=3, padding=1, use_batchnorm=use_batchnorm), 
)
 def forward(self, x):
x, skip = x
x = F.interpolate(x, scale_factor=2, mode='nearest')
 if skip is not None:
x = torch.cat([x, skip], dim=1)
 x = self.attention1(x)
 x = self.block(x)
 x = self.attention2(x)
 return x
class CenterBlock(DecoderBlock):
def forward(self, x):
return self.block(x)
 class UnetDecoder(Model):
def __init__(
 self,
encoder_channels,
decoder_channels=(256, 128, 64, 32, 16),
final_channels=1,
use_batchnorm=True,
center=False,
attention_type=None 
):
super().__init__()
if center:
channels = encoder_channels[0]
self.center = CenterBlock(channels, channels, use_batchnorm=use_batchnorm)
 else:
self.center = None
in_channels = self.compute_channels(encoder_channels, decoder_channels)
 out_channels = decoder_channels
self.layer1 = DecoderBlock(in_channels[0], out_channels[0],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer2 = DecoderBlock(in_channels[1], out_channels[1],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer3 = DecoderBlock(in_channels[2], out_channels[2],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer4 = DecoderBlock(in_channels[3], out_channels[3],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer5 = DecoderBlock(in_channels[4], out_channels[4],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.final_conv = nn.Conv2d(out_channels[4], final_channels, kernel_size=(1, 1))
self.initialize()
def compute_channels(self, encoder_channels, decoder_channels):
channels = [
encoder_channels[0] + encoder_channels[1],
encoder_channels[2] + decoder_channels[0],
encoder_channels[3] + decoder_channels[1],
encoder_channels[4] + decoder_channels[2],
0 + decoder_channels[3],
]
return channels
def forward(self, x):
encoder_head = x[0]
skips = x[1:]
if self.center:
encoder_head = self.center(encoder_head)
 x = self.layer1([encoder_head, skips[0]])
 x = self.layer2([x, skips[1]])
 x = self.layer3([x, skips[2]])
 x = self.layer4([x, skips[3]])
 x = self.layer5([x, None])
 x = self.final_conv(x)
 return x
class Unet(EncoderDecoder):

 def __init__(
 self,
encoder_name='resnet34',
encoder_weights='imagenet',
decoder_use_batchnorm=True,
decoder_channels=(256, 128, 64, 32, 16),
classes=1,
activation='sigmoid',
center=False,
attention_type=None 
):
encoder = get_encoder(
 encoder_name,
encoder_weights=encoder_weights 
)
 decoder = UnetDecoder(
 encoder_channels=encoder.out_shapes,
decoder_channels=decoder_channels,
final_channels=classes,
use_batchnorm=decoder_use_batchnorm,
center=center,
attention_type=attention_type 
)
 super().__init__(encoder, decoder, activation)
 self.name = 'u-{}'.format(encoder_name )<choose_model_class>
test_pred = mdl_xgb.predict(data=d_test )
<choose_model_class><EOS> 
submissions = pd.DataFrame()
submissions['id'] = real_test.iloc[:, 0]
submissions['target'] = test_pred
submissions.to_csv('xgb_model.csv', index=False )