Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<categorify>
from pathlib import Path
import numpy as np
import pandas as pd
import lightgbm as lgb
import os
from sklearn.model_selection import KFold
from tqdm.notebook import tqdm
import seaborn as sbn
from datetime import datetime, timedelta
from sklearn import datasets, linear_model
import matplotlib.pyplot as plt
import statsmodels.api as sm
import random
import tensorflow as tf
from tensorflow import keras
from scipy.stats import norm
data =(
 src.transform(tfms,size=224)
.databunch(bs=bs)
.normalize(imagenet_stats)
)
 data<train_model>
pd.options.mode.chained_assignment = None
learn.fit_one_cycle(4, 1e-2 )<train_model>
path = Path('/kaggle/input/osic-pulmonary-fibrosis-progression')
 assert path.exists()
learn.fit_one_cycle(10, max_lr=slice(1e-6,1e-3))<predict_on_test>
TRAIN_TYPES={"Patient": "category",
"Weeks": "int16", "FVC": "int32", 'Percent': 'float32', "Age": "uint8",
"Sex": "category", "SmokingStatus": "category" }
SUBMISSION_TYPES={"Patient_Week": "category", "FVC": "int32", "Confidence": "int16"}
def read_data(path):
train_df = pd.read_csv(path/'train.csv', dtype = TRAIN_TYPES)
 test_df = pd.read_csv(path/'test.csv', dtype = TRAIN_TYPES)
 submission_df = pd.read_csv(path/'sample_submission.csv', dtype = SUBMISSION_TYPES)
 train_df.drop_duplicates(keep='first', inplace=True, subset=['Patient','Weeks'])
 return train_df, test_df, submission_df
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )<import_modules>
train_df, test_df, submission_df = read_data(path )
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json<compute_test_metric>
def prepare_submission(df, test_df):
df['Patient'] = df['Patient_Week'].apply(lambda x:x.split('_')[0])
 df['Weeks'] = df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
df = df[['Patient','Weeks','Confidence','Patient_Week']]
df = df.merge(test_df.drop('Weeks', axis=1 ).copy() , on=['Patient'])
 return df
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<train_model>
submission_df = prepare_submission(submission_df, test_df )
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )<load_from_csv>
def adapt_percent_in_submission() :
previous_match = None
for i, r in submission_df.iterrows() :
in_training = train_df[(train_df['Patient'] == r['Patient'])&(train_df['Weeks'] == r['Weeks'])]
if(len(in_training)> 0):
previous_match = in_training['Percent'].item()
submission_df.iloc[i, submission_df.columns.get_loc('Percent')] = previous_match
elif previous_match is not None:
submission_df.iloc[i, submission_df.columns.get_loc('Percent')] = previous_match
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()<save_to_csv>
train_df['WHERE'] = 'train'
test_df['WHERE'] = 'val'
submission_df['WHERE'] = 'test'
data = train_df.append([test_df, submission_df] )
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
preds,y = learn.TTA(ds_type=DatasetType.Test)
 test_predictions = optR.predict(preds, coefficients)
 sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
sample_df.to_csv('submission.csv',index=False )<define_variables>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
seed = 5
np.random.seed(seed)
 tf.set_random_seed(seed )<set_options>
base = data.loc[data.Weeks == data.min_week]
print(os.listdir('.. /input'))
%matplotlib inline
IMG_SIZE=256
BATCH_SIZE = 16<import_modules>
base = base[['Patient','FVC', 'Percent']].copy()
base.columns = ['Patient','min_FVC', 'min_Percent']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base
MEAN_RGB = [0.485 * 255, 0.456 * 255, 0.406 * 255]
STDDEV_RGB = [0.229 * 255, 0.224 * 255, 0.225 * 255]
MAP_INTERPOLATION_TO_ORDER = {
"nearest": 0,
"bilinear": 1,
"biquadratic": 2,
"bicubic": 3,
}
def center_crop_and_resize(image, image_size, crop_padding=32, interpolation="bicubic"):
assert image.ndim in {2, 3}
assert interpolation in MAP_INTERPOLATION_TO_ORDER.keys()
h, w = image.shape[:2]
padded_center_crop_size = int(
(image_size /(image_size + crop_padding)) * min(h, w)
)
 offset_height =(( h - padded_center_crop_size)+ 1)// 2
offset_width =(( w - padded_center_crop_size)+ 1)// 2
image_crop = image[
offset_height : padded_center_crop_size + offset_height,
offset_width : padded_center_crop_size + offset_width,
]
resized_image = resize(
 image_crop, 
(image_size, image_size),
order=MAP_INTERPOLATION_TO_ORDER[interpolation],
preserve_range=True, 
)
 return resized_image
def preprocess_input(x):
assert x.ndim in(3, 4)
 assert x.shape[-1] == 3
x = x - np.array(MEAN_RGB)
 x = x / np.array(STDDEV_RGB)
 return x
class EfficientConv2DKernelInitializer(Initializer):

 def __call__(self, shape, dtype=K.floatx() , **kwargs):
kernel_height, kernel_width, _, out_filters = shape
fan_out = int(kernel_height * kernel_width * out_filters)
 return tf.random_normal(
 shape, mean=0.0, stddev=np.sqrt(2.0 / fan_out), dtype=dtype 
)
 class EfficientDenseKernelInitializer(Initializer):

 def __call__(self, shape, dtype=K.floatx() , **kwargs):

 init_range = 1.0 / np.sqrt(shape[1])
 return tf.random_uniform(shape, -init_range, init_range, dtype=dtype)
 conv_kernel_initializer = EfficientConv2DKernelInitializer()
dense_kernel_initializer = EfficientDenseKernelInitializer()
get_custom_objects().update(
 {
"EfficientDenseKernelInitializer": EfficientDenseKernelInitializer,
"EfficientConv2DKernelInitializer": EfficientConv2DKernelInitializer,
} 
)
 class Swish(KL.Layer):
def call(self, inputs):
return tf.nn.swish(inputs)
 class DropConnect(KL.Layer):
def __init__(self, drop_connect_rate=0.0, **kwargs):
super().__init__(**kwargs)
 self.drop_connect_rate = drop_connect_rate
def call(self, inputs, training=None):
def drop_connect() :
keep_prob = 1.0 - self.drop_connect_rate
batch_size = tf.shape(inputs)[0]
random_tensor = keep_prob
random_tensor += tf.random_uniform(
 [batch_size, 1, 1, 1], dtype=inputs.dtype 
)
 binary_tensor = tf.floor(random_tensor)
 output = tf.div(inputs, keep_prob)* binary_tensor
return output
return K.in_train_phase(drop_connect, inputs, training=training)
 def get_config(self):
config = super().get_config()
config["drop_connect_rate"] = self.drop_connect_rate
return config
get_custom_objects().update({"DropConnect": DropConnect, "Swish": Swish})
 IMAGENET_WEIGHTS = {
"efficientnet-b0": {
"name": "efficientnet-b0_imagenet_1000.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b0_imagenet_1000.h5",
"md5": "bca04d16b1b8a7c607b1152fe9261af7",
},
"efficientnet-b0-notop": {
"name": "efficientnet-b0_imagenet_1000_notop.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b0_imagenet_1000_notop.h5",
"md5": "45d2f3b6330c2401ef66da3961cad769",
},
"efficientnet-b1": {
"name": "efficientnet-b1_imagenet_1000.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b1_imagenet_1000.h5",
"md5": "bd4a2b82f6f6bada74fc754553c464fc",
},
"efficientnet-b1-notop": {
"name": "efficientnet-b1_imagenet_1000_notop.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b1_imagenet_1000_notop.h5",
"md5": "884aed586c2d8ca8dd15a605ec42f564",
},
"efficientnet-b2": {
"name": "efficientnet-b2_imagenet_1000.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b2_imagenet_1000.h5",
"md5": "45b28b26f15958bac270ab527a376999",
},
"efficientnet-b2-notop": {
"name": "efficientnet-b2_imagenet_1000_notop.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b2_imagenet_1000_notop.h5",
"md5": "42fb9f2d9243d461d62b4555d3a53b7b",
},
"efficientnet-b3": {
"name": "efficientnet-b3_imagenet_1000.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b3_imagenet_1000.h5",
"md5": "decd2c8a23971734f9d3f6b4053bf424",
},
"efficientnet-b3-notop": {
"name": "efficientnet-b3_imagenet_1000_notop.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b3_imagenet_1000_notop.h5",
"md5": "1f7d9a8c2469d2e3d3b97680d45df1e1",
},
"efficientnet-b4": {
"name": "efficientnet-b4_imagenet_1000.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b4_imagenet_1000.h5",
"md5": "01df77157a86609530aeb4f1f9527949",
},
"efficientnet-b4-notop": {
"name": "efficientnet-b4_imagenet_1000_notop.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b4_imagenet_1000_notop.h5",
"md5": "e7c3b780f050f8f49c800f23703f285c",
},
"efficientnet-b5": {
"name": "efficientnet-b5_imagenet_1000.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b5_imagenet_1000.h5",
"md5": "c31311a1a38b5111e14457145fccdf32",
},
"efficientnet-b5-notop": {
"name": "efficientnet-b5_imagenet_1000_notop.h5",
"url": "https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b5_imagenet_1000_notop.h5",
"md5": "a09b36129b41196e0bb659fd84fbdd5f",
},
}
GlobalParams = collections.namedtuple(
 "GlobalParams",
[
"batch_norm_momentum",
"batch_norm_epsilon",
"dropout_rate",
"data_format",
"num_classes",
"width_coefficient",
"depth_coefficient",
"depth_divisor",
"min_depth",
"drop_connect_rate",
], 
)
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs = collections.namedtuple(
 "BlockArgs",
[
"kernel_size",
"num_repeat",
"input_filters",
"output_filters",
"expand_ratio",
"id_skip",
"strides",
"se_ratio",
], 
)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 def efficientnet_params(model_name):

 params_dict = {
"efficientnet-b0":(1.0, 1.0, 224, 0.2),
"efficientnet-b1":(1.0, 1.1, 240, 0.2),
"efficientnet-b2":(1.1, 1.2, 260, 0.3),
"efficientnet-b3":(1.2, 1.4, 300, 0.3),
"efficientnet-b4":(1.4, 1.8, 380, 0.4),
"efficientnet-b5":(1.6, 2.2, 456, 0.4),
"efficientnet-b6":(1.8, 2.6, 528, 0.5),
"efficientnet-b7":(2.0, 3.1, 600, 0.5),
}
return params_dict[model_name]
class BlockDecoder(object):

 def _decode_block_string(self, block_string):

 assert isinstance(block_string, str)
 ops = block_string.split("_")
 options = {}
for op in ops:
splits = re.split(r"(\d.*)", op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
if "s" not in options or len(options["s"])!= 2:
raise ValueError("Strides options should be a pair of integers.")
 return BlockArgs(
 kernel_size=int(options["k"]),
num_repeat=int(options["r"]),
input_filters=int(options["i"]),
output_filters=int(options["o"]),
expand_ratio=int(options["e"]),
id_skip=("noskip" not in block_string),
se_ratio=float(options["se"])if "se" in options else None,
strides=[int(options["s"][0]), int(options["s"][1])], 
)
 def _encode_block_string(self, block):

 args = [
"r%d" % block.num_repeat,
"k%d" % block.kernel_size,
"s%d%d" %(block.strides[0], block.strides[1]),
"e%s" % block.expand_ratio,
"i%d" % block.input_filters,
"o%d" % block.output_filters,
]
if block.se_ratio > 0 and block.se_ratio <= 1:
args.append("se%s" % block.se_ratio)
 if block.id_skip is False:
args.append("noskip")
 return "_".join(args)
 def decode(self, string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(self._decode_block_string(block_string))
return blocks_args
def encode(self, blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(self._encode_block_string(block))
return block_strings
def efficientnet(
 width_coefficient=None,
depth_coefficient=None,
dropout_rate=0.2,
drop_connect_rate=0.2, 
):

 blocks_args = [
"r1_k3_s11_e1_i32_o16_se0.25",
"r2_k3_s22_e6_i16_o24_se0.25",
"r2_k5_s22_e6_i24_o40_se0.25",
"r3_k3_s22_e6_i40_o80_se0.25",
"r3_k5_s11_e6_i80_o112_se0.25",
"r4_k5_s22_e6_i112_o192_se0.25",
"r1_k3_s11_e6_i192_o320_se0.25",
]
global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
data_format="channels_last",
num_classes=1000,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None, 
)
 decoder = BlockDecoder()
return decoder.decode(blocks_args), global_params
def get_model_params(model_name, override_params=None):

 if model_name.startswith("efficientnet"):
width_coefficient, depth_coefficient, input_shape, dropout_rate = efficientnet_params(
 model_name 
)
 blocks_args, global_params = efficientnet(
 width_coefficient, depth_coefficient, dropout_rate 
)
 else:
raise NotImplementedError("model name is not pre-defined: %s" % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params, input_shape
__all__ = [
"EfficientNet",
"EfficientNetB0",
"EfficientNetB1",
"EfficientNetB2",
"EfficientNetB3",
"EfficientNetB4",
"EfficientNetB5",
"EfficientNetB6",
"EfficientNetB7",
]
def round_filters(filters, global_params):

 orig_f = filters
multiplier = global_params.width_coefficient
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
if not multiplier:
return filters
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def SEBlock(block_args, global_params):
num_reduced_filters = max(1, int(block_args.input_filters * block_args.se_ratio))
filters = block_args.input_filters * block_args.expand_ratio
if global_params.data_format == "channels_first":
channel_axis = 1
spatial_dims = [2, 3]
else:
channel_axis = -1
spatial_dims = [1, 2]
def block(inputs):
x = inputs
x = KL.Lambda(lambda a: K.mean(a, axis=spatial_dims, keepdims=True))(x)
 x = KL.Conv2D(
 num_reduced_filters,
kernel_size=[1, 1],
strides=[1, 1],
kernel_initializer=conv_kernel_initializer,
padding="same",
use_bias=True,
)(x)
 x = Swish()(x)
 x = KL.Conv2D(
 filters,
kernel_size=[1, 1],
strides=[1, 1],
kernel_initializer=conv_kernel_initializer,
padding="same",
use_bias=True,
)(x)
 x = KL.Activation("sigmoid" )(x)
 out = KL.Multiply()([x, inputs])
 return out
return block
def MBConvBlock(block_args, global_params, drop_connect_rate=None):
batch_norm_momentum = global_params.batch_norm_momentum
batch_norm_epsilon = global_params.batch_norm_epsilon
if global_params.data_format == "channels_first":
channel_axis = 1
spatial_dims = [2, 3]
else:
channel_axis = -1
spatial_dims = [1, 2]
has_se =(
(block_args.se_ratio is not None)
 and(block_args.se_ratio > 0)
 and(block_args.se_ratio <= 1)
)
 filters = block_args.input_filters * block_args.expand_ratio
kernel_size = block_args.kernel_size
def block(inputs):
if block_args.expand_ratio != 1:
x = KL.Conv2D(
 filters,
kernel_size=[1, 1],
strides=[1, 1],
kernel_initializer=conv_kernel_initializer,
padding="same",
use_bias=False,
)(inputs)
 x = KL.BatchNormalization(
 axis=channel_axis,
momentum=batch_norm_momentum,
epsilon=batch_norm_epsilon,
)(x)
 x = Swish()(x)
 else:
x = inputs
x = KL.DepthwiseConv2D(
 [kernel_size, kernel_size],
strides=block_args.strides,
depthwise_initializer=conv_kernel_initializer,
padding="same",
use_bias=False,
)(x)
 x = KL.BatchNormalization(
 axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon
)(x)
 x = Swish()(x)
 if has_se:
x = SEBlock(block_args, global_params )(x)
 x = KL.Conv2D(
 block_args.output_filters,
kernel_size=[1, 1],
strides=[1, 1],
kernel_initializer=conv_kernel_initializer,
padding="same",
use_bias=False,
)(x)
 x = KL.BatchNormalization(
 axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon
)(x)
 if block_args.id_skip:
if(
 all(s == 1 for s in block_args.strides)
 and block_args.input_filters == block_args.output_filters 
):
if drop_connect_rate:
x = DropConnect(drop_connect_rate )(x)
 x = KL.Add()([x, inputs])
 return x
return block
def EfficientNet(
 input_shape, block_args_list, global_params, input_tensor=None, include_top=True, pooling=None 
):
batch_norm_momentum = global_params.batch_norm_momentum
batch_norm_epsilon = global_params.batch_norm_epsilon
if global_params.data_format == "channels_first":
channel_axis = 1
else:
channel_axis = -1
if input_tensor is None:
inputs = KL.Input(shape=input_shape)
 else:
if not K.is_keras_tensor(input_tensor):
inputs = KL.Input(tensor=input_tensor, shape=input_shape)
 else:
inputs = input_tensor
x = inputs
x = KL.Conv2D(
 filters=round_filters(32, global_params),
kernel_size=[3, 3],
strides=[2, 2],
kernel_initializer=conv_kernel_initializer,
padding="same",
use_bias=False,
)(x)
 x = KL.BatchNormalization(
 axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon
)(x)
 x = Swish()(x)
 block_idx = 1
n_blocks = sum([block_args.num_repeat for block_args in block_args_list])
 drop_rate = global_params.drop_connect_rate or 0
drop_rate_dx = drop_rate / n_blocks
for block_args in block_args_list:
assert block_args.num_repeat > 0
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, global_params),
output_filters=round_filters(block_args.output_filters, global_params),
num_repeat=round_repeats(block_args.num_repeat, global_params), 
)
 x = MBConvBlock(
 block_args, global_params, drop_connect_rate=drop_rate_dx * block_idx
)(x)
 block_idx += 1
if block_args.num_repeat > 1:
block_args = block_args._replace(
 input_filters=block_args.output_filters, strides=[1, 1] 
)
 for _ in xrange(block_args.num_repeat - 1):
x = MBConvBlock(
 block_args, global_params, drop_connect_rate=drop_rate_dx * block_idx
)(x)
 block_idx += 1
x = KL.Conv2D(
 filters=round_filters(1280, global_params),
kernel_size=[1, 1],
strides=[1, 1],
kernel_initializer=conv_kernel_initializer,
padding="same",
use_bias=False,
)(x)
 x = KL.BatchNormalization(
 axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon
)(x)
 x = Swish()(x)
 if include_top:
x = KL.GlobalAveragePooling2D(data_format=global_params.data_format )(x)
 if global_params.dropout_rate > 0:
x = KL.Dropout(global_params.dropout_rate )(x)
 x = KL.Dense(
 global_params.num_classes, kernel_initializer=dense_kernel_initializer
)(x)
 x = KL.Activation("softmax" )(x)
 else:
if pooling == "avg":
x = KL.GlobalAveragePooling2D(data_format=global_params.data_format )(x)
 elif pooling == "max":
x = KL.GlobalMaxPooling2D(data_format=global_params.data_format )(x)
 outputs = x
model = KM.Model(inputs, outputs)
 return model
def _get_model_by_name(
 model_name,
input_shape=None,
input_tensor=None,
include_top=True,
weights=None,
classes=1000,
pooling=None 
):

 if weights not in {None, "imagenet"}:
raise ValueError('Parameter `weights` should be one of [None, "imagenet"]')
 if weights == "imagenet" and model_name not in IMAGENET_WEIGHTS:
raise ValueError(
 "There are not pretrained weights for {} model.".format(model_name)
)
 if weights == "imagenet" and include_top and classes != 1000:
raise ValueError(
 "If using `weights` and `include_top`" " `classes` should be 1000" 
)
 block_agrs_list, global_params, default_input_shape = get_model_params(
 model_name, override_params={"num_classes": classes} 
)
 if input_shape is None:
input_shape =(default_input_shape, default_input_shape, 3)
 model = EfficientNet(
 input_shape,
block_agrs_list,
global_params,
input_tensor=input_tensor,
include_top=include_top,
pooling=pooling, 
)
 model.name = model_name
if weights:
if not include_top:
weights_name = model_name + "-notop"
else:
weights_name = model_name
weights = IMAGENET_WEIGHTS[weights_name]
weights_path = get_file(
 weights["name"],
weights["url"],
cache_subdir="models",
md5_hash=weights["md5"], 
)
 model.load_weights(weights_path)
 return model
def EfficientNetB0(
 include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None 
):
return _get_model_by_name(
 "efficientnet-b0",
include_top=include_top,
input_shape=input_shape,
input_tensor=input_tensor,
weights=weights,
classes=classes,
pooling=pooling, 
)
 def EfficientNetB1(
 include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None 
):
return _get_model_by_name(
 "efficientnet-b1",
include_top=include_top,
input_shape=input_shape,
input_tensor=input_tensor,
weights=weights,
classes=classes,
pooling=pooling, 
)
 def EfficientNetB2(
 include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None 
):
return _get_model_by_name(
 "efficientnet-b2",
include_top=include_top,
input_shape=input_shape,
input_tensor=input_tensor,
weights=weights,
classes=classes,
pooling=pooling, 
)
 def EfficientNetB3(
 include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None 
):
return _get_model_by_name(
 "efficientnet-b3",
include_top=include_top,
input_shape=input_shape,
input_tensor=input_tensor,
weights=weights,
classes=classes,
pooling=pooling, 
)
 def EfficientNetB4(
 include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None 
):
return _get_model_by_name(
 "efficientnet-b4",
include_top=include_top,
input_shape=input_shape,
input_tensor=input_tensor,
weights=weights,
classes=classes,
pooling=pooling, 
)
 def EfficientNetB5(
 include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None 
):
return _get_model_by_name(
 "efficientnet-b5",
include_top=include_top,
input_shape=input_shape,
input_tensor=input_tensor,
weights=weights,
classes=classes,
pooling=pooling, 
)
 def EfficientNetB6(
 include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None 
):
return _get_model_by_name(
 "efficientnet-b6",
include_top=include_top,
input_shape=input_shape,
input_tensor=input_tensor,
weights=weights,
classes=classes,
pooling=pooling, 
)
 def EfficientNetB7(
 include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None 
):
return _get_model_by_name(
 "efficientnet-b7",
include_top=include_top,
input_shape=input_shape,
input_tensor=input_tensor,
weights=weights,
classes=classes,
pooling=pooling, 
)
 EfficientNetB0.__doc__ = _get_model_by_name.__doc__
EfficientNetB1.__doc__ = _get_model_by_name.__doc__
EfficientNetB2.__doc__ = _get_model_by_name.__doc__
EfficientNetB3.__doc__ = _get_model_by_name.__doc__
EfficientNetB4.__doc__ = _get_model_by_name.__doc__
EfficientNetB5.__doc__ = _get_model_by_name.__doc__
EfficientNetB6.__doc__ = _get_model_by_name.__doc__
EfficientNetB7.__doc__ = _get_model_by_name.__doc__<load_from_csv>
data = data.merge(base, on='Patient', how='left' )
train_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv' )<count_values>
data['base_week'] = data['Weeks'] - data['min_week']
data['base_week'] = data['base_week']
del base
train_df['diagnosis'].value_counts()<prepare_x_and_y>
data[data['Patient'] == 'ID00421637202311550012437']
N = train_df.shape[0]
x_train = np.empty(( N, IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(train_df['id_code'])) :
x_train[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/train_images/{image_id}.png'
)<categorify>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
y_train = pd.get_dummies(train_df['diagnosis'] ).values
print(y_train.shape)
<categorify>
data = data.rename(columns={"Age": "age", "min_FVC": "BASE", "base_week": "week", "Percent": "percent"})
 FE += ['age','week','BASE', 'percent']
FE
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))<split>
train_df = data.loc[data.WHERE=='train']
test_df = data.loc[data.WHERE=='val']
submission_df = data.loc[data.WHERE=='test']
del data
x_train, x_val, y_train, y_val = train_test_split(
 x_train, y_train_multi,
test_size=0.15,
random_state=2019
)<feature_engineering>
train_df.sort_values(['Patient', 'Weeks'], inplace=True )
x_val=x_val/255<data_type_conversions>
y = train_df['FVC']
y
train_df['diagnosis']=train_df['diagnosis'].astype(str )<set_options>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )
datagen = ImageDataGenerator(
 zoom_range=0.6,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
vertical_flip=True,
rotation_range=360,
width_shift_range=0.1,
height_shift_range=0.1,
rescale=1./255
)<define_variables>
seed_everything(42 )
data_generator = datagen.flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019 )<define_variables>
C1_val = 70
C2_val = 1000
C1, C2 = C1_val, C2_val
q = np.array([0.2, 0.50, 0.8])
 def score(y_true, y_pred):
sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = np.max(sigma)
 delta = np.abs(y_true[:, 0] - fvc_pred)
 delta = np.min(delta)
 sq2 = np.sqrt(2.)
metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 def qloss(y_true, y_pred):
print('y_true.shape', y_true.shape)
 print('y_pred.shape', y_pred.shape)
 e = y_true - y_pred
v = np.max(q*e,(q-1)*e)
 return np.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
y_true = y_true.reshape(-1, 1)
 return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def tilted_loss(q, y, f):
e =(y - f)
 return keras.backend.mean(keras.backend.maximum(q * e,(q - 1)* e), axis=-1 )
true_labels = np.array([1, 0, 1, 1, 0, 1])
 pred_labels = np.array([1, 0, 0, 0, 0, 1] )<compute_test_metric>
n_estimators = 30000
if not 'sub_row' in locals() :
sub_row = 0.75
if not 'bagging_freq' in locals() :
bagging_freq = 1
if not 'learning_rate' in locals() :
learning_rate = 0.4
leave_size = 4
lgb_params = {
"objective": 'quantile',
'n_jobs': 1,
'max_depth': leave_size + 1,
'num_leaves': 2**leave_size-1,
"min_data_in_leaf": 2**(leave_size + 1)-1,
"n_estimators": n_estimators,
'learning_rate': 8e-3,
'colsample_bytree': 0.9,
'boosting_type': 'gbdt',
"early_stopping_rounds": 100,
'verbosity': 1000,
"metric": ["rmse", "mse"]
}
cat_feats = ['Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']
accuracy_score(true_labels, pred_labels )<compute_test_metric>
optimizer = tf.optimizers.Adam(learning_rate=0.001)
 early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=20)
 EPOCHS = 200
BATCH_SIZE = 32
UNITS = 75
cohen_kappa_score(true_labels, pred_labels )<compute_train_metric>
def keras_pred(x_train, train_labels, q):
print(q)
 train_labels = np.array(train_labels.values, dtype='float32')
 input_dim = x_train.shape[1]
model = keras.Sequential([
keras.layers.Dense(UNITS, input_dim=input_dim),
keras.layers.Activation('relu'),
keras.layers.Dense(UNITS, activation=tf.nn.relu),
keras.layers.Dense(1)
 ])
 model.compile(loss=lambda y, f: tilted_loss(q, y, f), optimizer=optimizer)
 model.fit(x_train, train_labels, epochs=EPOCHS, batch_size=BATCH_SIZE,
verbose=1, validation_split=0.2, callbacks=[early_stop])
 return model
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val = y_val.sum(axis=1)- 1
y_pred = self.model.predict(X_val)> 0.5
y_pred = y_pred.astype(int ).sum(axis=1)- 1
_val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 model.save_weights('model.h5')
 model_json = model.to_json()
with open('model.json', "w")as json_file:
json_file.write(model_json)
 json_file.close()
return<choose_model_class>
def ols_quantile(m, X, q):
mean_pred = m.predict(X)
 se = np.sqrt(m.scale)
 return mean_pred + norm.ppf(q)* se
efficient = EfficientNetB5(
 weights=None,
include_top=False,
input_shape=(IMG_SIZE,IMG_SIZE,3)
 )<load_pretrained>
ensemble_weights = [4./11, 1./11, 6./11]
assert 1.0 - np.sum(ensemble_weights)< 0.00001
efficient.load_weights('.. /input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5' )<choose_model_class>
NFOLD = 5
kf = KFold(n_splits=NFOLD, shuffle=False)
 pred = {a: np.zeros(( train_df.shape[0])) for a in q.tolist() }
def build_model() :
model = Sequential()
model.add(efficient)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=0.00005),
metrics=['accuracy'] 
)
 return model<train_on_grid>
models = []
linear_models = []
keras_models = []
for cnt,(tr_idx, val_idx)in tqdm(enumerate(kf.split(X)) , total=NFOLD):
X_train, y_train = X.loc[tr_idx], y.loc[tr_idx]
X_valid, y_valid = X.loc[val_idx], y.loc[val_idx]
print(f"FOLD {cnt}", X_train.shape, y_train.shape, X_valid.shape, y_valid.shape)
 for qi, quantile_alpha in enumerate(q.tolist()):
lgb_params['alpha'] = quantile_alpha
m_lgb_regressor = lgb.LGBMRegressor(**lgb_params)
 m_lgb_regressor.fit(X=X_train, y=y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)],
eval_names=['train mloss', 'valid mloss'],
eval_metric=lgb_params['metric'],
verbose=lgb_params['verbosity'],
early_stopping_rounds=lgb_params["early_stopping_rounds"],
categorical_feature=cat_feats)
 lin_model = sm.RLM(y_train, X_train, M=sm.robust.norms.AndrewWave() ).fit()
lin_predict = ols_quantile(lin_model, X_valid, quantile_alpha)
 keras_model = keras_pred(X_train, y_train, quantile_alpha)
 keras_predict = keras_model.predict(X_valid ).reshape(-1)
 lgb_predict = m_lgb_regressor.predict(X_valid)
 pred[quantile_alpha][val_idx] = np.average([lin_predict, lgb_predict, keras_predict], axis = 0, weights=ensemble_weights)
 models.append(m_lgb_regressor)
 linear_models.append(lin_model)
 keras_models.append(keras_model )
kappa_metrics = Metrics()
est=EarlyStopping(monitor='val_loss',patience=5, min_delta=0.005)
 call_backs=[est,kappa_metrics]
history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
validation_data=(x_val,y_val),
epochs=20,
callbacks=call_backs )<predict_on_test>
full_preds = np.vstack([pred[a] for a in q] ).T
score(np.array(y ).reshape(-1, 1), full_preds )
model.load_weights('model.h5')
 test_pred=[]
for i,img_code in enumerate(tqdm(test_df['id_code'])) :
img=preprocess_image(f'.. /input/aptos2019-blindness-detection/test_images/{img_code}.png')
 pred=model.predict(img[np.newaxis]/255)>0.5
pred=pred.astype(int ).sum(axis=1)- 1
test_pred.append(pred)
<save_to_csv>
pred = []
for i in range(NFOLD):
cur_pred = []
for j, quantile in enumerate(q):
model_idx = i * 3 + j
model = models[model_idx]
lin_predict = ols_quantile(linear_models[model_idx], submission_df[FE], quantile)
 dbmc_predict = model.predict(submission_df[FE])
 keras_predict = keras_models[model_idx].predict(submission_df[FE] ).reshape(-1)
 cur_pred.append(np.average([lin_predict, dbmc_predict, keras_predict], axis = 0, weights=ensemble_weights))
pred.append(np.array(cur_pred ).T )
test_pred=np.concatenate(test_pred,axis=0)
 test_pred=pd.Series(test_pred)
 test_df['diagnosis'] = test_pred
test_df.to_csv('submission.csv',index=False)
 print(test_df.head() )<import_modules>
submission_df['FVC1'] = final_preds[:,1]
submission_df['Confidence1'] = final_preds[:, 2] - final_preds[:, 0]
from pathlib import Path
import numpy as np
import pandas as pd
import cv2<load_from_csv>
submission_df[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )
<load_from_csv><EOS> 
submission_final_df = pd.read_csv("submission.csv" )