Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<categorify>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
y_cat = to_categorical(y)
 print(y_cat.shape )<choose_model_class>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
model = Sequential()
model.add(Dense(256, input_dim=108, init='uniform', activation='relu'))
model.add(Dense(128, init='normal', activation='sigmoid'))
model.add(Dense(99, activation='softmax'))<choose_model_class>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
model = Sequential()
model.add(Dense(512,input_dim=192, init='uniform', activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(256, activation='sigmoid'))
model.add(Dropout(0.3))
model.add(Dense(99, activation='softmax'))<choose_model_class>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
model.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"] )<train_model>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
history = model.fit(X,y_cat,batch_size=192,
nb_epoch=130,verbose=0, validation_split=0.1 )<load_from_csv>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
test = pd.read_csv('.. /input/test.csv' )<drop_column>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
index = test.pop('id' )<normalization>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(0.4 )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
test = StandardScaler().fit(test ).transform(test )<predict_on_test>
tr_p, vl_p = train_test_split(P,
shuffle=True,
train_size= 0.8 )
yPred = model.predict_proba(test )<create_dataframe>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
yPred = pd.DataFrame(yPred,index=index,columns=sort(parent_data.species.unique()))<save_to_csv>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
fp = open('submission_nn_kernel.csv','w')
 fp.write(yPred.to_csv() )<define_variables>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
sys.path = [
'.. /input/ttach-kaggle/ttach/',
] + sys.path 
<categorify>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
def get_tiles(img, tile_size, n_tiles, mode=0):
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(
 img,
[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2, pad_w - pad_w // 2], [0, 0]],
constant_values=255, 
)
 img3 = img2.reshape(
 img2.shape[0] // tile_size, tile_size, img2.shape[1] // tile_size, tile_size, 3 
)
 img3 = img3.transpose(0, 2, 1, 3, 4 ).reshape(-1, tile_size, tile_size, 3)
 n_tiles_with_info =(
 img3.reshape(img3.shape[0], -1 ).sum(1)< tile_size ** 2 * 3 * 255
).sum()
if len(img3)< n_tiles:
img3 = np.pad(
 img3,
[[0, n_tiles - len(img3)], [0, 0], [0, 0], [0, 0]],
constant_values=255, 
)
 idxs = np.argsort(img3.reshape(img3.shape[0], -1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
return img3, n_tiles_with_info >= n_tiles
def concat_tiles(tiles, tile_shape):
image = cv2.hconcat([cv2.vconcat(tiles[ts])for ts in tile_shape])
 return image<create_dataframe>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
class TestDataset(Dataset):
def __init__(
 self,
df,
image_dir,
num_tile,
img_size,
tile_size_r,
tile_size_k,
tile_mode,
transform, 
):
self.df = df
self.img_ids = self.df.image_id
self.image_dir = image_dir
self.transform = transform
self.num_tiles = num_tile
self.img_size = img_size
self.tile_mode = tile_mode
self.tile_size_k = tile_size_k
self.tile_size_r = tile_size_r
self.is_karolinska =(self.df.data_provider == "karolinska" ).values
self.tiff_level = 1
n = int(self.num_tiles ** 0.5)
 self.tile_shape = np.array(range(self.num_tiles)).reshape(( n, n))
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
file_name = self.img_ids[idx]
file_path = os.path.join(self.image_dir, f"{file_name}.tiff")
 img_tmp = skimage.io.MultiImage(file_path)[self.tiff_level]
tiles, _ = get_tiles(
 img_tmp,
tile_size=self.tile_size_k,
n_tiles=self.num_tiles,
mode=self.tile_mode, 
)
 tiles = np.array(tiles)
 image = concat_tiles(tiles, tile_shape=self.tile_shape)
 image = image.astype(np.float32)
 image = 255 - image
image /= 255
if self.transform:
augmented = self.transform(image=image)
 image = augmented["image"]
return image<train_model>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
GlobalParams = collections.namedtuple(
 "GlobalParams",
[
"batch_norm_momentum",
"batch_norm_epsilon",
"dropout_rate",
"num_classes",
"width_coefficient",
"depth_coefficient",
"depth_divisor",
"min_depth",
"drop_connect_rate",
"image_size",
], 
)
 BlockArgs = collections.namedtuple(
 "BlockArgs",
[
"kernel_size",
"num_repeat",
"input_filters",
"output_filters",
"expand_ratio",
"id_skip",
"stride",
"se_ratio",
], 
)
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 class SwishImplementation(torch.autograd.Function):
@staticmethod
def forward(ctx, i):
result = i * torch.sigmoid(i)
 ctx.save_for_backward(i)
 return result
@staticmethod
def backward(ctx, grad_output):
i = ctx.saved_variables[0]
sigmoid_i = torch.sigmoid(i)
 return grad_output *(sigmoid_i *(1 + i *(1 - sigmoid_i)))
 class MemoryEfficientSwish(nn.Module):
def forward(self, x):
return SwishImplementation.apply(x)
 class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 if not training:
return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand(
 [batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device 
)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(
 self,
in_channels,
out_channels,
kernel_size,
stride=1,
dilation=1,
groups=1,
bias=True, 
):
super().__init__(
 in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias 
)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(
 x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2] 
)
 return F.conv2d(
 x,
self.weight,
self.bias,
self.stride,
self.padding,
self.dilation,
self.groups, 
)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(
 self, in_channels, out_channels, kernel_size, image_size=None, **kwargs 
):
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw = image_size if type(image_size)== list else [image_size, image_size]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(
(pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2)
)
 else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(
 x,
self.weight,
self.bias,
self.stride,
self.padding,
self.dilation,
self.groups, 
)
 return x
class Identity(nn.Module):
def __init__(self,):
super(Identity, self ).__init__()
def forward(self, input):
return input 
<init_hyperparams>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
def efficientnet_params(model_name):

 params_dict = {
"efficientnet-b0":(1.0, 1.0, 224, 0.2),
"efficientnet-b1":(1.0, 1.1, 240, 0.2),
"efficientnet-b2":(1.1, 1.2, 260, 0.3),
"efficientnet-b3":(1.2, 1.4, 300, 0.3),
"efficientnet-b4":(1.4, 1.8, 380, 0.4),
"efficientnet-b5":(1.6, 2.2, 456, 0.4),
"efficientnet-b6":(1.8, 2.6, 528, 0.5),
"efficientnet-b7":(2.0, 3.1, 600, 0.5),
"efficientnet-b8":(2.2, 3.6, 672, 0.5),
"efficientnet-l2":(4.3, 5.3, 800, 0.5),
}
return params_dict[model_name]
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split("_")
 options = {}
for op in ops:
splits = re.split(r"(\d.*)", op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert("s" in options and len(options["s"])== 1)or(
 len(options["s"])== 2 and options["s"][0] == options["s"][1] 
)
 return BlockArgs(
 kernel_size=int(options["k"]),
num_repeat=int(options["r"]),
input_filters=int(options["i"]),
output_filters=int(options["o"]),
expand_ratio=int(options["e"]),
id_skip=("noskip" not in block_string),
se_ratio=float(options["se"])if "se" in options else None,
stride=[int(options["s"][0])], 
)
 @staticmethod
def _encode_block_string(block):

 args = [
"r%d" % block.num_repeat,
"k%d" % block.kernel_size,
"s%d%d" %(block.strides[0], block.strides[1]),
"e%s" % block.expand_ratio,
"i%d" % block.input_filters,
"o%d" % block.output_filters,
]
if 0 < block.se_ratio <= 1:
args.append("se%s" % block.se_ratio)
 if block.id_skip is False:
args.append("noskip")
 return "_".join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(
 width_coefficient=None,
depth_coefficient=None,
dropout_rate=0.2,
drop_connect_rate=0.2,
image_size=None,
num_classes=1000, 
):

 blocks_args = [
"r1_k3_s11_e1_i32_o16_se0.25",
"r2_k3_s22_e6_i16_o24_se0.25",
"r2_k5_s22_e6_i24_o40_se0.25",
"r3_k3_s22_e6_i40_o80_se0.25",
"r3_k5_s11_e6_i80_o112_se0.25",
"r4_k5_s22_e6_i112_o192_se0.25",
"r1_k3_s11_e6_i192_o320_se0.25",
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
num_classes=num_classes,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None,
image_size=image_size, 
)
 return blocks_args, global_params
def get_model_params(model_name, override_params):

 if model_name.startswith("efficientnet"):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s 
)
 else:
raise NotImplementedError("model name is not pre-defined: %s" % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
url_map = {"no-url"}
url_map_advprop = {"no-url"}
def load_pretrained_weights(model, model_name, load_fc=True, advprop=False):

 url_map_ = url_map_advprop if advprop else url_map
state_dict = model_zoo.load_url(url_map_[model_name])
 if load_fc:
model.load_state_dict(state_dict)
 else:
state_dict.pop("_fc.weight")
 state_dict.pop("_fc.bias")
 res = model.load_state_dict(state_dict, strict=False)
 assert set(res.missing_keys)== set(
 ["_fc.weight", "_fc.bias"] 
), "issue loading pretrained weights"
print("Loaded pretrained weights for {}".format(model_name))
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(
 0 < self._block_args.se_ratio <= 1 
)
 self.id_skip = block_args.id_skip
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 inp = self._block_args.input_filters
oup =(
 self._block_args.input_filters * self._block_args.expand_ratio 
)
 if self._block_args.expand_ratio != 1:
self._expand_conv = Conv2d(
 in_channels=inp, out_channels=oup, kernel_size=1, bias=False 
)
 self._bn0 = nn.BatchNorm2d(
 num_features=oup, momentum=self._bn_mom, eps=self._bn_eps 
)
 k = self._block_args.kernel_size
s = self._block_args.stride
self._depthwise_conv = Conv2d(
 in_channels=oup,
out_channels=oup,
groups=oup,
kernel_size=k,
stride=s,
bias=False, 
)
 self._bn1 = nn.BatchNorm2d(
 num_features=oup, momentum=self._bn_mom, eps=self._bn_eps 
)
 if self.has_se:
num_squeezed_channels = max(
 1, int(self._block_args.input_filters * self._block_args.se_ratio)
)
 self._se_reduce = Conv2d(
 in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1 
)
 self._se_expand = Conv2d(
 in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1 
)
 final_oup = self._block_args.output_filters
self._project_conv = Conv2d(
 in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False 
)
 self._bn2 = nn.BatchNorm2d(
 num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps 
)
 self._swish = MemoryEfficientSwish()
def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = self._swish(self._bn0(self._expand_conv(inputs)))
 x = self._swish(self._bn1(self._depthwise_conv(x)))
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(self._swish(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters =(
 self._block_args.input_filters,
self._block_args.output_filters, 
)
 if(
 self.id_skip
and self._block_args.stride == 1
and input_filters == output_filters 
):
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish() 
<choose_model_class>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), "blocks_args should be a list"
assert len(blocks_args)> 0, "block args must be greater than 0"
self._global_params = global_params
self._blocks_args = blocks_args
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(
 32, self._global_params 
)
 self._conv_stem = Conv2d(
 in_channels, out_channels, kernel_size=3, stride=2, bias=False 
)
 self._bn0 = nn.BatchNorm2d(
 num_features=out_channels, momentum=bn_mom, eps=bn_eps 
)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(
 block_args.input_filters, self._global_params 
),
output_filters=round_filters(
 block_args.output_filters, self._global_params 
),
num_repeat=round_repeats(block_args.num_repeat, self._global_params), 
)
 self._blocks.append(MBConvBlock(block_args, self._global_params))
if block_args.num_repeat > 1:
block_args = block_args._replace(
 input_filters=block_args.output_filters, stride=1 
)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(
 num_features=out_channels, momentum=bn_mom, eps=bn_eps 
)
 self._avg_pooling = nn.AdaptiveAvgPool2d(1)
 self._dropout = nn.Dropout(self._global_params.dropout_rate)
 self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 self._swish = MemoryEfficientSwish()
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
for block in self._blocks:
block.set_swish(memory_efficient)
 def extract_features(self, inputs):

 x = self._swish(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = self._swish(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 bs = inputs.size(0)
 x = self.extract_features(inputs)
 x = self._avg_pooling(x)
 x = x.view(bs, -1)
 x = self._dropout(x)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 return cls(blocks_args, global_params)
 @classmethod
def from_pretrained(
 cls, model_name, advprop=False, num_classes=1000, in_channels=3 
):
model = cls.from_name(model_name, override_params={"num_classes": num_classes})
 load_pretrained_weights(
 model, model_name, load_fc=(num_classes == 1000), advprop=advprop 
)
 if in_channels != 3:
Conv2d = get_same_padding_conv2d(image_size=model._global_params.image_size)
 out_channels = round_filters(32, model._global_params)
 model._conv_stem = Conv2d(
 in_channels, out_channels, kernel_size=3, stride=2, bias=False 
)
 return model
@classmethod
def get_image_size(cls, model_name):
cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name):

 valid_models = ["efficientnet-b" + str(i)for i in range(9)]
if model_name not in valid_models:
raise ValueError("model_name should be one of: " + ", ".join(valid_models))
class CustomEfficientNet(nn.Module):
def __init__(
 self,
base="efficientnet-b0",
pool_type="gem",
in_ch=3,
out_ch=1,
pretrained=False, 
):
super(CustomEfficientNet, self ).__init__()
assert base in {
"efficientnet-b0",
"efficientnet-b1",
"efficientnet-b2",
"efficientnet-b3",
"efficientnet-b4",
}
assert pool_type in {"concat", "avg", "gem"}
self.base = base
self.in_ch = in_ch
self.out_ch = out_ch
self.pretrained = pretrained
if pretrained:
self.net = EfficientNet.from_pretrained(base)
 else:
self.net = EfficientNet.from_name(base)
 out_shape = self.net._fc.in_features
if pool_type == "concat":
self.net._avg_pooling = AdaptiveConcatPool2d()
out_shape = out_shape * 2
elif pool_type == "gem":
self.net._avg_pooling = GeM()
out_shape = out_shape
self.net._fc = nn.Sequential(
 Flatten() , SEBlock(out_shape), nn.Dropout() , nn.Linear(out_shape, out_ch)
)
 def forward(self, x):
x = self.net(x)
 return x
class AdaptiveConcatPool2d(nn.Module):
def __init__(self, sz=None):
super().__init__()
sz = sz or(1, 1)
 self.ap = nn.AdaptiveAvgPool2d(sz)
 self.mp = nn.AdaptiveMaxPool2d(sz)
 def forward(self, x):
return torch.cat([self.mp(x), self.ap(x)], 1)
 class SEBlock(nn.Module):
def __init__(self, in_ch, r=8):
super(SEBlock, self ).__init__()
self.linear_1 = nn.Linear(in_ch, in_ch // r)
 self.linear_2 = nn.Linear(in_ch // r, in_ch)
 def forward(self, x):
input_x = x
x = F.relu(self.linear_1(x), inplace=True)
 x = self.linear_2(x)
 x = torch.sigmoid(x)
 x = input_x * x
return x
class Flatten(nn.Module):
def forward(self, input):
return input.view(input.size(0), -1)
 def gem(x, p=3, eps=1e-6):
return F.avg_pool2d(x.clamp(min=eps ).pow(p),(x.size(-2), x.size(-1)) ).pow(1.0 / p)
 class GeM(nn.Module):
def __init__(self, p=3, eps=1e-6):
super(GeM, self ).__init__()
self.p = Parameter(torch.ones(1)* p)
 self.eps = eps
def forward(self, x):
return gem(x, p=self.p, eps=self.eps)
 def __repr__(self):
return(
 self.__class__.__name__
+ f"(p={self.p.data.tolist() [0]:.4f}, eps={str(self.eps)})" 
)
 def inference(
 dataloader, model, device, out_ch, tta=False, fp16=False, dataloader2=None, 
):
probs = []
def extract_preds(logits_tmp):
if out_ch == 1:
preds_tmp = logits_tmp
elif out_ch == 5:
preds_tmp = logits_tmp.sigmoid().sum(1)
 elif out_ch > 5:
preds_tmp = logits_tmp[:, :5].sigmoid()
preds_tmp = preds_tmp.sum(1)
 return preds_tmp.detach()
for i, images in tqdm(enumerate(dataloader), total=len(dataloader)) :
if fp16:
images = images.half()
images = images.to(device)
 with torch.no_grad() :
logits = model(images)
 preds = extract_preds(logits)
 if tta:
logits_hflip = model(images.flip(3))
preds_hflip = extract_preds(logits_hflip)
 preds =(preds + preds_hflip)/ 2.0
probs.append(preds.to("cpu" ).numpy())
 probs = np.concatenate(probs)
 if dataloader2 is not None:
probs2 = inference(
 dataloader2, model, device, out_ch, tta, fp16=fp16, dataloader2=None 
)
 return(probs + probs2)/ 2.0, probs
return probs
def quadratic_weighted_kappa(y_hat, y):
return cohen_kappa_score(y_hat, y, weights="quadratic")
 class MyOptimizedRounder:
def __init__(self):
self.coef = None
self.initial_coef = np.array([0.5, 1.5, 2.5, 3.5, 4.5])
 def judge_each_thres(self, thres, x):
tmp = thres.reshape(( 1, -1)) <= x.reshape(( -1, 1))
result = tmp.sum(axis=1)
 return result
def _kappa_loss(self, coef, x, y):
x_p = self.judge_each_thres(coef, np.copy(x))
ll = quadratic_weighted_kappa(y, x_p)
 return -ll
def fit(self, x, y):
loss_partial = partial(self._kappa_loss, x=x, y=y)
 self.coef = sp.optimize.minimize(
 loss_partial, self.initial_coef, method="nelder-mead" 
)
 def predict(self, x, coef=None):
if coef is None:
coef = self.coef["x"]
return self.judge_each_thres(coef, np.copy(x))
def predict_default_thres(self, x):
coef = self.initial_coef.copy()
return self.judge_each_thres(coef, np.copy(x))
def coefficients(self):
return self.coef["x"]
def print_eval_local(df, preds):
ground_truth = df.isup_grade.values
score = quadratic_weighted_kappa(ground_truth, preds)
 print(f"Local CV kappa score(all): {score}")
 indx_val_karolinska = df.data_provider == "karolinska"
indx_val_radboud = df.data_provider == "radboud"
indx = indx_val_karolinska
v_acc_k =(preds[indx] == ground_truth[indx] ).mean() * 100.0
v_kappa_k = quadratic_weighted_kappa(ground_truth[indx], preds[indx])
 print(f"Local CV kappa score(karolinska): {v_kappa_k}")
 print(f"Local CV accuracy(karolinska): {v_acc_k}")
 indx = indx_val_radboud
v_acc_r =(preds[indx] == ground_truth[indx] ).mean() * 100.0
v_kappa_r = quadratic_weighted_kappa(ground_truth[indx], preds[indx])
 print(f"Local CV kappa score(radboud): {v_kappa_r}")
 print(f"Local CV accuracy(radboud): {v_acc_r}")
<init_hyperparams>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
class Config:
def __init__(self, on_kernel=True, kfold=1, debug=False):
self.on_kernel = on_kernel
self.num_tile = 64
self.tile_size_k = 192
self.tile_size_r = 192
self.img_size = 1536
self.fp16 = False
self.batch_size = 4
if self.fp16:
self.batch_size = int(self.batch_size * 2)
 self.tta = False
self.ada_bn = False
self.in_ch = 3
self.out_ch = 10
self.base = "efficientnet-b1"
self.pseudo_use_threshold = 0.2
self.pseudo_soft_label = False
self.weight_name = "final_2_efficientnet-b1_kfold_{}_latest.pt"
self.weight_name = self.weight_name.format(kfold)
 if on_kernel:
self.data_dir = ".. /input/prostate-cancer-grade-assessment"
test_image_folder = os.path.join(self.data_dir, "test_images")
 train_image_folder = os.path.join(self.data_dir, "train_images")
 is_test = os.path.exists(test_image_folder)
 self.image_folder = test_image_folder if is_test else train_image_folder
df_train = pd.read_csv(os.path.join(self.data_dir, "train.csv"))
sample = pd.read_csv(os.path.join(self.data_dir, "test.csv"))
self.df = sample if is_test else df_train.loc[:16]
else:
self.data_dir = ".. /input"
self.image_folder = ".. /input/train_images"
df = pd.read_csv(".. /input/train-5kfold.csv")
 self.kfold = 1
self.df = df[df["kfold"] == self.kfold].reset_index(drop=True)
 if debug:
self.df = self.df[:16].reset_index(drop=True)
 def get_weight_path(self):
if self.on_kernel:
return os.path.join(".. /input/030-weight", self.weight_name)
 else:
dir_name = self.weight_name.split("_")[0]
return os.path.join(".. /output/model", dir_name, self.weight_name)
 def get_args_ds(self):
return {
"df": self.df,
"image_dir": self.image_folder,
"num_tile": self.num_tile,
"img_size": self.img_size,
"tile_size_k": self.tile_size_k,
"tile_size_r": self.tile_size_r,
"transform": Compose([ToTensorV2() ]),
}
def get_args_dl(self):
return {"batch_size": self.batch_size, "shuffle": False, "num_workers": 4}
def get_model(cfg_model):
net = None
if "seresnext" in cfg_model.base:
net = CustomSEResNeXt(
 in_ch=cfg_model.in_ch,
out_ch=cfg_model.out_ch,
pool_type="gem",
pretrained=False, 
)
 elif "efficientnet" in cfg_model.base:
net = CustomEfficientNet(
 base=cfg_model.base,
in_ch=cfg_model.in_ch,
out_ch=cfg_model.out_ch,
pretrained=False, 
)
 assert net is not None
return net
def inference_from_cfg(cfg, model=None):
print("Make frame and path")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 test_dataset = TestDataset(tile_mode=0, **cfg.get_args_ds())
 test_loader = DataLoader(test_dataset, **cfg.get_args_dl())
 test_dataset2 = TestDataset(tile_mode=2, **cfg.get_args_ds())
 test_loader2 = DataLoader(test_dataset2, **cfg.get_args_dl())
 print("Make data loader")
 if model is not None:
print(f"Use existed Model: {cfg.base}")
 else:
print(f"Get Model: {cfg.base}")
 model = get_model(cfg)
 w_path = cfg.get_weight_path()
print(f"Load weight...: {w_path}")
 state_dict = torch.load(w_path)["state_dict"]
state_dict = {k[4:]: v for k, v in state_dict.items() }
model.load_state_dict(state_dict)
 if cfg.ada_bn:
for m in model.modules() :
if isinstance(m, _BatchNorm):
m.track_running_stats = False
if cfg.fp16:
model = model.half()
model.eval()
model = model.to(device)
 transforms = tta.Compose(
 [tta.HorizontalFlip() , tta.VerticalFlip() , tta.Rotate90(angles=[0, 90, 180])] 
)
 model = tta.ClassificationTTAWrapper(model, transforms)
 print("Inference start")
 probs, probs_without_tta = inference(
 test_loader,
model,
device,
out_ch=cfg.out_ch,
tta=cfg.tta,
fp16=cfg.fp16,
dataloader2=test_loader2, 
)
 return probs, probs_without_tta
def inference_mean_cfgs(cfgs):
probs_all = None
probs_without_tta_all = None
for c in cfgs:
probs, probs_without_tta = inference_from_cfg(c)
 if probs_all is None:
probs_all = probs
probs_without_tta_all = probs_without_tta
else:
probs_all = probs_all + probs
probs_without_tta_all = probs_without_tta_all + probs_without_tta
probs = probs_all / float(len(cfgs))
probs_without_tta = probs_without_tta_all / float(len(cfgs))
return probs, probs_without_tta
def main(on_kernel=False, debug=False):
cfgs = [Config(on_kernel=on_kernel, kfold=k, debug=debug)for k in [1, 4, 5]]
probs_raw, probs_without_tta_raw = inference_mean_cfgs(cfgs)
 print("Round predicts")
 rounder = MyOptimizedRounder()
preds = rounder.predict_default_thres(probs_raw)
 preds_without_tta = rounder.predict_default_thres(probs_without_tta_raw)
 if on_kernel:
df = cfgs[0].df[["image_id"]].copy()
df["probs_raw"] = probs_raw
df["isup_grade"] = preds
df["isup_grade"] = df["isup_grade"].astype(int)
 return df
else:
print(f"
*** Normal preds results ***")
 print_eval_local(cfgs[0].df, preds_without_tta)
 print(f"
*** Mode TTA preds results ***")
 print_eval_local(cfgs[0].df, preds)
 df = cfgs[0].df
df["probs_raw"] = probs_raw
df["probs_without_tta_raw"] = probs_without_tta_raw
df["preds"] = preds
df["preds_without_tta"] = preds_without_tta
df.to_csv(f"local_preds_{cfgs[0].weight_name.split('.')[0]}.csv", index=False)
<define_variables>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path<define_variables>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
sys.path = [
'.. /input/ttach-kaggle/ttach/',
] + sys.path 
<import_modules>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
import skimage.io
import numpy as np
import pandas as pd
import torchvision
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from efficientnet_pytorch import model as enet
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook as tqdm<load_from_csv>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
data_dir = '.. /input/prostate-cancer-grade-assessment'
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
model_dir = '.. /input/panda-public-models'
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 df = df_test if is_test else df_train.loc[:16]
tile_size = 256
image_size = 256
n_tiles = 36
batch_size = 4
num_workers = 4
device = torch.device('cuda')
 print(image_folder )<load_pretrained>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
class enetv3(nn.Module):
def __init__(self, modelname, out_dim=5, freeze_bn=True):
super(enetv3, self ).__init__()
if "resnet" in modelname or "resnext" in modelname:
basemodel = resnet.__dict__[modelname](
 pretrained=True)
 if modelname == "rx50":
basemodel = torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext50_32x4d_ssl')
 if "eff" in modelname:
self.basemodel = EfficientNet.from_name(modelname)
 self.myfc = nn.Linear(self.basemodel._fc.in_features, out_dim)
 self.basemodel._fc = nn.Identity()
self.basemodel._avg_pooling = nn.AdaptiveAvgPool2d(1)
 else:
fcs = basemodel.fc.state_dict() ["weight"].shape[1]
self.basemodel = nn.Sequential(*list(basemodel.children())[:-2])
 if poolmethod=="maxpool":
self.pool = nn.AdaptiveMaxPool2d(1)
 else:
self.pool = nn.AdaptiveAvgPool2d(1)
 self.myfc = nn.Linear(fcs, out_dim)
 def extract(self, x):
return self.basemodel(x)
 def forward(self, x):
x = self.basemodel(x)
 if not "eff" in modelname:
x = self.pool(x ).squeeze(2 ).squeeze(2)
 x = self.myfc(x)
 else:
x = self.myfc(x)
 return x
def load_models(model_files):
models = []
for model_f in model_files:
model_f = os.path.join(".. /input/latesubspanda", model_f)
 if "res" in model_f:
model = enetv2(out_dim=5)
 elif "eff" in model_f:
model = enetv3(modelname, out_dim=5)
 model.load_state_dict(torch.load(model_f))
model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models
modelname="efficientnet-b0"
model_files = [
'efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold0.pth',
]
model_files2 = [
'efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold0.pth',
"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold1.pth",
"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold2.pth",
"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold3.pth",
"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold4.pth"
]
models2 = load_models(model_files2)
 models = load_models(model_files )<categorify>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img)< n_tiles:
img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False,
sub_imgs=False 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.sub_imgs = sub_imgs
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(image_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1][:,:,::-1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
idxes = np.asarray(idxes)+ self.n_tiles if self.sub_imgs else idxes
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 return torch.tensor(images )<load_pretrained>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
dataset = PANDADataset(df, image_size, n_tiles, 0)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False )<choose_model_class>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
if len(dataset)> 50:
transforms = tta.Compose(
 [
tta.HorizontalFlip() ,
tta.VerticalFlip() ,
tta.Rotate90(angles=[0, 90, 180]),
] 
)
 else:
transforms = tta.Compose(
 [
] 
)
 tta_models = []
tta_models2 = []
for model in models:
tta_models.append(tta.ClassificationTTAWrapper(model, transforms))
for model in models2:
tta_models2.append(tta.ClassificationTTAWrapper(model, transforms))<define_search_model>
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
LOGITS = []
LOGITS2 = []
LOGITS12 = []
LOGITS22 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 for tta_model in tta_models:
logits = tta_model(data)
 LOGITS.append(logits)
 for i, tta_model in enumerate(tta_models2):
if i == 0:
logits = tta_model(data ).sigmoid()
else:
logits = logits + tta_model(data ).sigmoid()
LOGITS2.append(logits/5)
 for data in tqdm(loader2):
data = data.to(device)
 for tta_model in tta_models:
logits = tta_model(data)
 LOGITS12.append(logits)
 for i, tta_model in enumerate(tta_models2):
if i == 0:
logits = tta_model(data ).sigmoid()
else:
logits = logits + tta_model(data ).sigmoid()
LOGITS22.append(logits/5)
 LOGITS =(torch.cat(LOGITS ).sigmoid().cpu() + torch.cat(LOGITS2 ).cpu()
+torch.cat(LOGITS12 ).sigmoid().cpu() + torch.cat(LOGITS22 ).cpu())/ 4<feature_engineering>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
PREDS = LOGITS.sum(1 ).numpy()
df['isup_grade'] = PREDS
df<rename_columns>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
df_tmp.rename({'probs_raw': 'part_1_pred'}, axis=1, inplace=True )<merge>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
df = df.merge(df_tmp[['image_id', 'part_1_pred']], on = 'image_id', how = 'left' )<feature_engineering>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
p_w = 0.5
df['isup_grade'] = df['isup_grade'] * p_w + df['part_1_pred'] *(1-p_w )<feature_engineering>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
df['isup_grade'] = df['isup_grade'].apply(lambda x: int(np.round(x)) )<save_to_csv>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )<define_variables>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
DEBUG = False<define_variables>
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.26*df1['Confidence'] + 0.74*df2['Confidence']
df.head()
<import_modules><EOS> 
df.to_csv('submission.csv', index=False )