Kaggle competition: iMet Collection 2019 - FGVC6
<SOS> metric: MeanFScoreBeta Kaggle data source: imet-2019-fgvc6<feature_engineering>
beginning_time = time()
def load_scan(path):
slices = [dicom.read_file(path + '/' + s)for s in os.listdir(path)]
slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))
try:
slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])
 except:
slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)
 for s in slices:
s.SliceThickness = slice_thickness
return slices<data_type_conversions>
tqdm_notebook.pandas()
%matplotlib inline
print(os.listdir(".. /input"))
def get_pixels_hu(slices):
image = np.stack([s.pixel_array for s in slices])
 image = image.astype(np.int16)
 image[image == -2000] = 0
for slice_number in range(len(slices)) :
intercept = slices[slice_number].RescaleIntercept
slope = slices[slice_number].RescaleSlope
if slope != 1:
image[slice_number] = slope * image[slice_number].astype(np.float64)
 image[slice_number] = image[slice_number].astype(np.int16)
 image[slice_number] += np.int16(intercept)
 return np.array(image, dtype=np.int16 )<count_duplicates>
print("chainercsv:", chainercv.__version__)
 print("chainer:", chainer.__version__ )
drop=train[train.duplicated(subset=['Patient','Weeks'],keep='last')]
print('No.of rows to be dropped:',drop.shape[0])
 train.drop_duplicates(subset=['Patient','Weeks'],keep='last',inplace=True )<feature_engineering>
DATA_DIR = ".. /input/imet-2019-fgvc6"
PRETRAINED_MODEL_DIR = ".. /input/chainercv-seresnet"
sub[['Patient','Weeks']]=sub.Patient_Week.str.split("_",expand = True)
 sub=sub[['Patient','Weeks','Confidence','Patient_Week']]
sub.head()<merge>
img_chmean_train = np.array([164.82181258, 155.93463791, 144.58968491], dtype="f")
sub=sub.merge(test.drop('Weeks',axis = 1),on="Patient")
 sub.head()<prepare_output>
seed = 1086
settings = OrderedDict(
 base_model="SEResNet152",
n_class=1103,
image_size=[128, 128],
max_epoch=20,
batch_size=128,
da_select=[
"random_distort",
"random_lr_flip",
"random_rotate",
"random_expand", "resize_with_random",
"random_crop"
],
learning_schedule="cosine",
epoch_per_cycle=20,
optimizer="NesterovAG",
learning_rate=0.01,
learning_rate_min=0.0001,
momentum=0.9,
weight_decay_rate=1e-04,
loss_func="FocalLoss",
alpha=0.5,
gamma=2, 
)
 settings["pretrained_model_path"] = "{}/{}".format(
 PRETRAINED_MODEL_DIR,
{
"SEResNet50": "se_resnet50_imagenet_converted_2018_06_25.npz",
"SEResNet101": "se_resnet101_imagenet_converted_2018_06_25.npz",
"SEResNet152": "se_resnet152_imagenet_converted_2018_06_25.npz",
}[settings["base_model"]] )
train['Dataset']='train'
sub['Dataset']='test'
data=train.append([sub])
 data.reset_index(inplace = True,drop=True)
 data.head()<categorify>
base_class = getattr(chainercv.links, settings["base_model"])
 class FeatureExtractor(base_class):

 def __init__(self, pretrained_model_path, extract_layers=["pool5"]):

 super(FeatureExtractor, self ).__init__(pretrained_model=pretrained_model_path)
 self._pick = extract_layers
self.remove_unused()
def __call__(self, x):

 h = x
for name in self.layer_names:
h = self[name](h)
 return h
class Ext2Linear(chainer.Chain):

 def __init__(self, n_class, extractor):

 super(Ext2Linear, self ).__init__()
with self.init_scope() :
self.extractor = extractor
self.fc = links.Linear(
 None, n_class, initialW=initializers.Normal(scale=0.01))
def __call__(self, x):

 return self.fc(self.extractor(x))
data = pd.concat([
data,
pd.get_dummies(data.Sex),
pd.get_dummies(data.SmokingStatus)
 ],axis=1)
 data.drop(['Sex','SmokingStatus'],axis=1,inplace=True)
 data['Weeks']=data['Weeks'].astype('int64')
 data.head()<data_type_conversions>
class MultiLabelClassifier(links.Classifier):

 def __init__(self, predictor, lossfun):

 super(MultiLabelClassifier, self ).__init__(predictor, lossfun)
 self.compute_accuracy = False
self.f_beta = None
self.metfun = self._fbeta_score
def __call__(self, x, t):

 loss = super().__call__(x, t)
 self.f_beta = None
self.f_beta = self.metfun(self.y, t)
 reporter.report({'f-beta': self.f_beta}, self)
 return loss
def _fbeta_score(self, y_pred, t, beta=2, th=0.2, epsilon=1e-09):

 y_prob = functions.sigmoid(y_pred ).data
t_pred =(y_prob >= th ).astype("i")
 true_pos =(t_pred * t ).sum(axis=1)
 pred_pos = t_pred.sum(axis=1)
 poss_pos = t.sum(axis=1)
 precision = true_pos /(pred_pos + epsilon)
 recall = true_pos /(poss_pos + epsilon)
 f_beta_each_id =(1 + beta ** 2)* precision * recall /(( beta ** 2)* precision + recall + epsilon)
 return functions.mean(f_beta_each_id )
def get_baseline(df):
_df=df.copy()
_df['min_week']=_df['Weeks']
_df.loc[_df.Dataset=='test','min_week']=0
_df["min_week"]=_df.groupby('Patient')['Weeks'].transform('min')
 _df['baselined_week']=_df['Weeks']-_df['min_week']
return _df
data['Weeks']=data['Weeks'].astype('int64')
 data=get_baseline(data)
 data.head()<merge>
class FocalLoss:

 def __init__(self, alpha=0.25, gamma=2):

 super(FocalLoss, self ).__init__()
self.alpha = alpha
self.gamma = gamma
def __call__(self, y_pred, t, epsilon=1e-31):

 p_dash = functions.clip(
(1 - t)+(2 * t - 1)* functions.sigmoid(y_pred), epsilon, 1 - epsilon)
 alpha_dash =(1 - t)+(2 * t - 1)* self.alpha
loss_by_sample_x_class = - alpha_dash *(1 - p_dash)** self.gamma * functions.log(p_dash)
 loss_by_sample = functions.sum(loss_by_sample_x_class, axis=1)
 return functions.mean(loss_by_sample )
def get_baseline_FVC(df):
_df = df.copy()
base = _df.loc[_df.Weeks == _df.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','base_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb == 1]
base.drop('nb', axis = 1, inplace = True)
 _df = _df.merge(base, on = 'Patient', how = 'left')
 _df.drop(['min_week'], axis = 1)
 return _df
data=get_baseline_FVC(data)
 data.head()<normalization>
def resize_pair(pair, size=settings["image_size"]):
img, label = pair
img = transforms.resize(img, size=size)
 return(img, label)
 def scale_and_subtract_mean(pair, mean_value=img_chmean_train):
img, label = pair
img =(img - mean_value[:, None, None])/ 255.
 return(img, label )
def scaling(series):
return(series-series.min())/(series.max() -series.min())
 data['Age']=scaling(data['Age'])
 data['Percent']=scaling(data['Percent'])
 data['baselined_week']=scaling(data['baselined_week'])
 data['base_FVC']=scaling(data['base_FVC'])
 data.head()<train_model>
class DataAugmentor() :

 def __init__(
 self, image_size, image_mean,
using_methods=["random_ud_flip","random_lr_flip", "random_90_rotate", "random_crop"] 
):

 self.img_size = image_size
self.img_mean = image_mean
self.img_mean_single = int(image_mean.mean())
 self.using_methods = using_methods
self.func_dict = {
"random_ud_flip": self._random_ud_flip,
"random_lr_flip": self._random_lr_flip,
"random_90_rotate": self._random_90_rotate,
"random_rotate": self._random_rotate,
"random_expand": self._random_expand,
"resize_with_random": self._resize_with_random,
"random_crop": self._random_crop,
"random_distort": random_distort,
}
self.da_funcs = [self.func_dict[um] for um in using_methods]
def __call__(self, pair):

 img_arr, label = pair
for func in self.da_funcs:
img_arr = func(img_arr)
 return img_arr, label
def _random_lr_flip(self, img_arr):

 if np.random.randint(2):
img_arr = img_arr[:, :, ::-1]
return img_arr
def _random_ud_flip(self, img_arr):

 if np.random.randint(2):
img_arr = img_arr[:, ::-1, :]
return img_arr
def _random_90_rotate(self, img_arr):

 if np.random.randint(2):
img_arr = img_arr.transpose(0, 2, 1)[:, ::-1, :]
return img_arr
def _random_rotate(self, img_arr, max_angle=10):

 angle = np.random.randint(-max_angle, max_angle + 1)
 if angle == 0:
return img_arr
return transforms.rotate(img_arr, angle, fill=self.img_mean_single)
 def _random_expand(self, img_arr):

 if np.random.randint(2):
return img_arr
return transforms.random_expand(img_arr, fill=self.img_mean)
 def _resize_with_random(self, img_arr):

 if img_arr.shape[-2:] == self.img_size:
return img_arr
return resize_with_random_interpolation(img_arr, self.img_size)
 def _random_crop(self, img_arr, rate=0.5):

 crop_size = self.img_size
resize_size = tuple(map(lambda x: int(x * 256 / 224), self.img_size))
if np.random.randint(2):
top = np.random.randint(0, resize_size[0] - crop_size[0])
 botom = top + crop_size[0]
left = np.random.randint(0, resize_size[1] - crop_size[1])
 right = left + crop_size[1]
img_arr = transforms.resize(img_arr, size=resize_size)[:, top:botom, left: right]
return img_arr
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):

 tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype = tf.float32))
metric =(delta / sigma_clip)* sq2 + tf.math.log(sigma_clip * sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):

 qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype = tf.float32)
 e = y_true - y_pred
v = tf.maximum(q * e,(q-1)* e)
 return K.mean(v)
 def mloss(_lambda):

 def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)* score(y_true, y_pred)
 return loss<choose_model_class>
class CosineShift(chainer.training.extension.Extension):

 def __init__(self, attr, value, period, period_mult=1, optimizer=None):
self._attr = attr
self._value = value
self._period = period
self._period_mult = period_mult
self._optimizer = optimizer
if not hasattr(self._value, '__getitem__'):
self._value =(self._value, 0)
 def initialize(self, trainer):
self._update_value(trainer)
 def __call__(self, trainer):
self._update_value(trainer)
 def _update_value(self, trainer):
optimizer = self._optimizer or trainer.updater.get_optimizer('main')
 epoch = trainer.updater.epoch
period_range = self._period
period_start = 0
period_end = period_range
while period_end <= epoch:
period_start = period_end
period_range *= self._period_mult
period_end += period_range
n_max, n_min = self._value
t_cur = epoch - period_start
t_i = period_range
value = n_min + 0.5 *(n_max - n_min)*(1 + cos(( t_cur / t_i)* pi))
setattr(optimizer, self._attr, value )
def make_model(nh):
z = Input(( nh,), name="Patient")
 x = Dense(100, activation="elu", name="d1" )(z)
 x = Dense(100, activation="elu", name="d3" )(x)
 p1 = Dense(3, activation="linear", name="p1" )(x)
 p2 = Dense(3, activation="elu", name="p2" )(x)
 preds = Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = Model(z,preds,name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
def predict(model, val_iter, gpu_device=-1):
val_pred_list = []
val_label_list = []
iter_num = 0
epoch_test_start = time()
while True:
val_batch = val_iter.next()
iter_num += 1
print("\rtmp_valid_iteration: {:0>5}".format(iter_num), end="")
 feature_val, label_val = chainer.dataset.concat_examples(val_batch, gpu_device)
 with chainer.no_backprop_mode() and chainer.using_config("train", False):
prediction_val = model(feature_val)
 val_pred_list.append(prediction_val)
 val_label_list.append(label_val)
 prediction_val.unchain_backward()
if val_iter.is_new_epoch:
print(" => valid end: {:.2f} sec".format(time() - epoch_test_start))
val_iter.epoch = 0
val_iter.current_position = 0
val_iter.is_new_epoch = False
val_iter._pushed_position = None
break
val_pred_all = cuda.to_cpu(functions.concat(val_pred_list, axis=0 ).data)
 val_label_all = cuda.to_cpu(functions.concat(val_label_list, axis=0 ).data)
 return val_pred_all, val_label_all
features_list=['baselined_week', 'Percent', 'Age', 'base_FVC', 'Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']
train=data.loc[data.Dataset == 'train']
sub=data.loc[data.Dataset == 'test']
y=train['FVC'].values.astype(float)
 X_train=train[features_list].values
X_test=sub[features_list].values
n_rows=X_train.shape[1]
train_preds=np.zeros(( X_train.shape[0], 3))
test_preds=np.zeros(( X_test.shape[0], 3))<train_model>
def average_fbeta_score(y_prob, t, th=0.2, beta=2, epsilon=1e-09):
t_pred =(y_prob >= th ).astype(int)
 true_pos =(t_pred * t ).sum(axis=1)
 pred_pos = t_pred.sum(axis=1)
 poss_pos = t.sum(axis=1)
 p_arr = true_pos /(pred_pos + epsilon)
 r_arr = true_pos /(poss_pos + epsilon)
 f_beta =(1 + beta ** 2)* p_arr * r_arr /(( beta ** 2)* p_arr + r_arr + epsilon)
 return f_beta.mean()
def search_best_threshold(y_prob, t, eval_func, search_range=[0.05, 0.95], interval=0.01):
tmp_th = search_range[0]
best_th = 0
best_eval = -(10**9 + 7)
 while tmp_th < search_range[1]:
eval_score = eval_func(y_prob, t, th=tmp_th)
 print(tmp_th, eval_score)
 if eval_score > best_eval:
best_th = tmp_th
best_eval = eval_score
tmp_th += interval
return best_th, best_eval
def make_pred_ids(test_pred, th):
class_array = np.arange(test_pred.shape[1])
 test_cond = test_pred >= th
pred_ids = [" ".join(map(str, class_array[cond])) for cond in test_cond]
return pred_ids
reduce_lr_loss=tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',factor=0.4,patience=150,verbose=0,epsilon=1e-4,mode='min')
 NFOLD = 6
kf = KFold(n_splits=NFOLD)
 OOF_val_score=[]
cnt = 0
BATCH_SIZE=128
EPOCHS = 800
for tr_idx, val_idx in kf.split(X_train):
cnt += 1
print(f"FOLD {cnt}")
 model=make_model(n_rows)
 history=model.fit(X_train[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(X_train[val_idx], y[val_idx]), verbose=0, callbacks=[reduce_lr_loss])
 print("train", model.evaluate(X_train[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", model.evaluate(X_train[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 train_preds[val_idx]=model.predict(X_train[val_idx],batch_size=BATCH_SIZE, verbose=0)
 OOF_val_score.append(model.evaluate(X_train[val_idx], y[val_idx], verbose = 0, batch_size = BATCH_SIZE, return_dict = True)['score'])
 print("predict test...")
 test_preds+=model.predict(X_test, batch_size=BATCH_SIZE, verbose=0)/NFOLD<compute_test_metric>
print("[end of setup]: {:.3f}".format(time() - beginning_time))
np.mean(OOF_val_score )<compute_test_metric>
train_df = pd.read_csv("{}/train.csv".format(DATA_DIR))
test_df = pd.read_csv("{}/sample_submission.csv".format(DATA_DIR))
labels_df = pd.read_csv("{}/labels.csv".format(DATA_DIR))
sigma_opt = mean_absolute_error(y, train_preds[:,1])
 sigma_uncertain = train_preds[:,2] - train_preds[:,0]
sigma_mean = np.mean(sigma_uncertain)
 print(sigma_opt, sigma_mean )<prepare_output>
%%time
train_attr_ohot = np.zeros(( len(train_df), len(labels_df)) , dtype="i")
 for idx, attr_arr in enumerate(
 train_df.attribute_ids.str.split(" " ).apply(lambda l: list(map(int, l)) ).values):
train_attr_ohot[idx, attr_arr] = 1
sub['FVC1'] = test_preds[:, 1]
sub['Confidence1'] = test_preds[:,2] - test_preds[:,0]
submission = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
submission.loc[~submission.FVC1.isnull() ].head(10 )<feature_engineering>
all_train_dataset = datasets.LabeledImageDataset(
 pairs=list(zip(( train_df.id + ".png" ).tolist() , train_attr_ohot)) ,
root="{}/train".format(DATA_DIR))
test_dataset = datasets.LabeledImageDataset(
 pairs=list(zip(( test_df.id + ".png" ).tolist() , [-1] * len(test_df))),
root="{}/test".format(DATA_DIR))
submission.loc[~submission.FVC1.isnull() ,'FVC'] = submission.loc[~submission.FVC1.isnull() ,'FVC1']
if sigma_mean < 70:
submission['Confidence'] = sigma_opt
else:
submission.loc[~submission.FVC1.isnull() ,'Confidence'] = submission.loc[~submission.FVC1.isnull() ,'Confidence1']<feature_engineering>
train_dataset, valid_dataset = datasets.split_dataset_random(
 all_train_dataset, first_size=int(len(all_train_dataset)* 0.8), seed=seed)
 print("train set:", len(train_dataset))
print("valid set:", len(valid_dataset))
org_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(org_test)) :
submission.loc[submission['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'FVC'] = org_test.FVC[i]
submission.loc[submission['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'Confidence'] = 70<save_to_csv>
train_dataset = datasets.TransformDataset(train_dataset, resize_pair)
 train_dataset = datasets.TransformDataset(
 train_dataset, DataAugmentor(
 image_size=settings["image_size"], image_mean=img_chmean_train,
using_methods=settings["da_select"])
)
 train_dataset = datasets.TransformDataset(train_dataset, scale_and_subtract_mean )
submission[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index = False )<import_modules>
valid_dataset = datasets.TransformDataset(valid_dataset, resize_pair)
 valid_dataset = datasets.TransformDataset(valid_dataset, scale_and_subtract_mean)
 test_dataset = datasets.TransformDataset(test_dataset, resize_pair)
 test_dataset = datasets.TransformDataset(test_dataset, scale_and_subtract_mean )
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error, mean_squared_error
from sklearn.model_selection import KFold,GroupKFold,TimeSeriesSplit,train_test_split
from time import time<import_modules>
print("[end of preparing data]: {:.3f}".format(time() - beginning_time))
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from keras.layers import LSTM, Dropout<drop_column>
model = Ext2Linear(
 settings["n_class"], FeatureExtractor(settings["pretrained_model_path"]))
train_model = MultiLabelClassifier(
 model, lossfun=FocalLoss(settings["alpha"], settings["gamma"]))
try:
del z_test
del y_test
del pred_test
print('deleted')
 except:
print('go' )<define_variables>
opt_class = getattr(optimizers, settings["optimizer"])
 if settings["optimizer"] != "Adam":
optimizer = opt_class(lr=settings["learning_rate"], momentum=settings["momentum"])
 optimizer.setup(train_model)
 optimizer.add_hook(chainer.optimizer.WeightDecay(settings["weight_decay_rate"]))
else:
optmizer = opt_class(
 alpha=settings["learning_rate"], weight_decay_rate=settings["weight_decay_rate"])
 optimizer.setup(train_model )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(24 )<define_variables>
train_iter = iterators.MultiprocessIterator(train_dataset, settings["batch_size"])
 valid_iter = iterators.MultiprocessIterator(
 valid_dataset, settings["batch_size"], repeat=False, shuffle=False )
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"<compute_test_metric>
updater = training.StandardUpdater(train_iter, optimizer, device=0)
 trainer = training.trainer.Trainer(updater, stop_trigger=(settings["max_epoch"], "epoch"), out="training_result" )
def laplace_log_likelihood(actual_fvc, predicted_fvc, confidence, return_values = False):

 sd_clipped = np.maximum(confidence, 70)
 delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)
 metric = - np.sqrt(2)* delta / sd_clipped - np.log(np.sqrt(2)* sd_clipped)
 if return_values:
return metric
else:
return np.mean(metric)
<feature_engineering>
print("[end of preparing trainer]: {:.3f}".format(time() - beginning_time))
tra = pd.read_csv(f"{ROOT}/train.csv")
 tra.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
<create_dataframe>
gc.collect()
tr = pd.DataFrame(tra ).copy()<concatenate>
%%time
trainer.run()
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
trained_model = trainer.updater.get_optimizer('main' ).target.predictor
serializers.save_npz('{}/epoch{:0>3}.model'.format("training_result", settings["max_epoch"]), trained_model )
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
<feature_engineering>
valid_iter = iterators.MultiprocessIterator(
 valid_dataset, settings["batch_size"], repeat=False, shuffle=False)
 val_pred, val_label = predict(trained_model, valid_iter, gpu_device=0 )
data['min_perc'] = data.Percent
data.loc[data.WHERE=='test','min_perc'] = np.nan
data['max_perc'] = data.Percent
data.loc[data.WHERE=='test','max_perc'] = np.nan
data['min_perc'] = data.groupby(['Age','SmokingStatus','Sex'])['Percent'].transform('min')
 data['max_perc'] = data.groupby(['Age','SmokingStatus','Sex'])['Percent'].transform('max')
 data['rif_perc'] = data.max_perc - data.min_perc
data.loc[data.rif_perc > 0, 'rif_perc']=(data.loc[data.rif_perc > 0, 'Percent']- data.loc[data.rif_perc > 0, 'min_perc'])/\ 
(data.loc[data.rif_perc > 0, 'max_perc'] - data.loc[data.rif_perc > 0, 'min_perc'])
 data = data.drop(['max_perc','min_perc'], axis=1)
<feature_engineering>
val_prob = functions.sigmoid(val_pred ).data
best_th, best_fbeta = search_best_threshold(
 val_prob, val_label, eval_func=average_fbeta_score, search_range=[0.1, 0.9], interval=0.01)
 print(best_th, best_fbeta )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<drop_column>
test_iter = iterators.MultiprocessIterator(
 test_dataset, settings["batch_size"], repeat=False, shuffle=False)
 test_pred, _ = predict(trained_model, test_iter, gpu_device=0 )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
test_prob = functions.sigmoid(test_pred ).data
test_pred_ids = make_pred_ids(test_prob, th=best_th )
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
test_df.to_csv("submission.csv", index=False )