Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<import_modules>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error, mean_squared_error
from sklearn.model_selection import KFold,GroupKFold,TimeSeriesSplit,train_test_split
from time import time
import os
import numpy as np
import pandas as pd
import pyarrow.parquet as pq
from joblib import Parallel, delayed
from tqdm import tqdm
import tensorflow as tf
from tensorflow import set_random_seed
from keras.layers import *
from keras.models import Model
from keras import backend as K
from keras import optimizers
from keras.callbacks import *
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.model_selection import train_test_split
from sklearn.metrics import matthews_corrcoef 
<define_variables>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from keras.layers import LSTM, Dropout
N_SPLITS = 5
sample_size = 800000
MAX_THREADS = 4
RANDOM_SEED = 2019<define_variables>
try:
del z_test
del y_test
del pred_test
print('deleted')
 except:
print('go' )
np.random.seed(RANDOM_SEED)
 set_random_seed(RANDOM_SEED )<normalization>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(24 )
class StatefullMCC(Layer):
def __init__(self, thresholds, **kwargs):
super(StatefullMCC, self ).__init__(**kwargs)
 self.thresholds = thresholds
self.stateful = True
self.name='matthews_correlation'
def reset_states(self):
K.get_session().run(tf.variables_initializer(self.local_variable))
def metric_variable(self, shape, dtype, validate_shape=True, name=None):
return tf.Variable(
 np.zeros(shape),
dtype=dtype,
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES],
validate_shape=validate_shape,
name=name, 
)
 def initialize_vars(self, y_true, y_pred):
self.tp = [None]*len(self.thresholds)
 self.tn = [None]*len(self.thresholds)
 self.fp = [None]*len(self.thresholds)
 self.fn = [None]*len(self.thresholds)
 for i,_ in enumerate(self.thresholds):
self.tp[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='tp%d'%i)
 self.tn[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='tn%d'%i)
 self.fp[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='fp%d'%i)
 self.fn[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='fn%d'%i)
 tp_op = tf.assign_add(self.tp[i], tf.count_nonzero(y_true * y_pred[i], axis=0))
self.add_update(tp_op)
 tn_op = tf.assign_add(self.tn[i], tf.count_nonzero(( 1-y_true)*(1-y_pred[i]), axis=0))
self.add_update(tn_op)
 fp_op = tf.assign_add(self.fp[i], tf.count_nonzero(( 1-y_true)*y_pred[i], axis=0))
self.add_update(fp_op)
 fn_op = tf.assign_add(self.fn[i], tf.count_nonzero(y_true*(1-y_pred[i]), axis=0))
self.add_update(fn_op)
 self.local_variable = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES)
 def __call__(self, y_true, y_pred):
rounded_preds = [K.cast(K.greater_equal(y_pred, t), 'float32')for t in self.thresholds]
self.initialize_vars(y_true, rounded_preds)
 mcc_vec = []
for i,_ in enumerate(self.thresholds):
num = tf.cast(( self.tp[i] * self.tn[i] - self.fp[i] * self.fn[i]), 'float32')
 den = K.sqrt(tf.cast(( self.tp[i] + self.fp[i])*(self.tp[i] + self.fn[i])*(self.tn[i] + self.fp[i])*(self.tn[i] + self.fn[i]), 'float32'))
mcc_vec.append(num/(den + tf.constant(K.epsilon())))
 return K.max(mcc_vec )<define_search_model>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.glorot_uniform(RANDOM_SEED)
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<load_from_csv>
def laplace_log_likelihood(actual_fvc, predicted_fvc, confidence, return_values = False):

 sd_clipped = np.maximum(confidence, 70)
 delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)
 metric = - np.sqrt(2)* delta / sd_clipped - np.log(np.sqrt(2)* sd_clipped)
 if return_values:
return metric
else:
return np.mean(metric)
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()<load_from_csv>
tra = pd.read_csv(f"{ROOT}/train.csv")
 tra.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
def get_features(dataset='train', split_parts=10):
if dataset == 'train':
cache_file = 'X.npy'
meta_file = '.. /input/metadata_train.csv'
elif dataset == 'test':
cache_file = 'X_test.npy'
meta_file = '.. /input/metadata_test.csv'
if os.path.isfile(cache_file):
X = np.load(cache_file)
 y = None
if dataset == 'train':
y = np.load('y.npy')
 else:
meta_df = pd.read_csv(meta_file)
 data_measurements = meta_df.pivot(index='id_measurement', columns='phase', values='signal_id')
 data_measurements = data_measurements.values
data_measurements = np.array_split(data_measurements, split_parts, axis=0)
 X = Parallel(n_jobs=min(split_parts, MAX_THREADS), verbose=1 )(delayed(prep_data )(p, dataset)for p in data_measurements)
 try:
y = meta_df.loc[meta_df['phase']==0, 'target'].values
except:
y = None
X = np.concatenate(X, axis=0)
 if dataset == 'train':
np.save("X.npy",X)
 np.save("y.npy",y)
 elif dataset == 'test':
np.save("X_test.npy",X)
 return X, y 
<define_variables>
tr = pd.DataFrame(tra ).copy()
max_num = 127
min_num = -128<categorify>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
ts_std =(ts - min_data)/(max_data - min_data)
 return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]<categorify>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in tqdm(range(0, sample_size, bucket_size)) :
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )<prepare_x_and_y>

 data['min_perc'] = data.Percent
data.loc[data.WHERE=='test','min_perc'] = np.nan
data['max_perc'] = data.Percent
data.loc[data.WHERE=='test','max_perc'] = np.nan
data['min_perc'] = data.groupby(['Age','SmokingStatus','Sex'])['Percent'].transform('min')
 data['max_perc'] = data.groupby(['Age','SmokingStatus','Sex'])['Percent'].transform('max')
 data['rif_perc'] = data.max_perc - data.min_perc
data.loc[data.rif_perc > 0, 'rif_perc']=(data.loc[data.rif_perc > 0, 'Percent']- data.loc[data.rif_perc > 0, 'min_perc'])/\ 
(data.loc[data.rif_perc > 0, 'max_perc'] - data.loc[data.rif_perc > 0, 'min_perc'])
 data = data.drop(['max_perc','min_perc'], axis=1)
def prep_data(signal_ids, dataset="train"):
signal_ids_all = np.concatenate(signal_ids)
 if dataset == "train":
praq_data = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in signal_ids_all] ).to_pandas()
elif dataset == "test":
praq_data = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in signal_ids_all] ).to_pandas()
else:
raise ValueError("Unknown dataset")
 X = []
for sids in tqdm(signal_ids):
data = praq_data[[str(s)for s in sids]].values.T
X_signal = [transform_ts(signal)for signal in data]
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 return X<prepare_x_and_y>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
%%time
X, y = get_features("train", split_parts=6 )<load_from_csv>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )<drop_column>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()<feature_engineering>
FE = []
COLS = ['Sex','SmokingStatus']
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
tmp = meta_test.reset_index()<define_variables>
true_mask = tmp["signal_id"].isin(exp_true ).values[::3]
false_mask = tmp["signal_id"].isin(exp_false ).values[::3]<feature_engineering>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['FVC_BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())


 FE += ['age','percent','week','FVC_BASE']
%%time
MAX_THREADS = 4
X_test_input, _ = get_features("test" )<drop_column>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
true_X_test = X_test_input[true_mask]
false_X_test = X_test_input[false_mask]<concatenate>
k=0
tr['KGroup'] = 0
for pat in tr.Patient.unique() :
k=k+1
tr.loc[tr.Patient == pat, 'KGroup'] = k
k=0
chunk['KGroup'] = 0
for pat in chunk.Patient.unique() :
k=k+1
chunk.loc[chunk.Patient == pat, 'KGroup'] = k
k=0
sub['KGroup'] = 0
for pat in sub.Patient.unique() :
k=k+1
sub.loc[sub.Patient == pat, 'KGroup'] = k
del k, pat
XX = np.append(X, true_X_test, axis=0)
 X = np.append(XX, false_X_test, axis=0 )<concatenate>
from tensorflow.keras import datasets, layers, models, regularizers, optimizers, callbacks
from tensorflow.keras.layers import Flatten, BatchNormalization
from keras.regularizers import l1_l2,l2,l1
from tensorflow.keras.optimizers import SGD
from tensorflow.keras import activations
from keras.layers import LeakyReLU
from tensorflow.keras.callbacks import History, EarlyStopping
y = np.append(y, [1] *(len(exp_true)// 3)+ [0]*(len(exp_false)// 3), axis=0 )<choose_model_class>
ALPHA= 0.01
L2 = 0.001
MOMENTUM = 0.9
DPO = 0.3
use_metric = 1.0
BATCH_SIZE = 16
HDO = 144
EPOCHS = 2000
OPTIMIZER= "adamax"
qmin = 0.2
qmed = 0.5
qmax = 0.8
CORRECTION = tf.constant(1, dtype='float32')
 GROUPS = True
NFOLD = 5
TESTSIZE = 0.0
VAL_SPLIT = 0.1
ERR = 0.0
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
init_glorot_uniform = initializers.glorot_uniform(seed=RANDOM_SEED)
 init_orthogonal = initializers.orthogonal(seed=RANDOM_SEED)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True, kernel_initializer=init_glorot_uniform, recurrent_initializer=init_orthogonal))(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True, kernel_initializer=init_glorot_uniform, recurrent_initializer=init_orthogonal))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu", kernel_initializer=init_glorot_uniform )(x)
 x = Dense(1, activation="sigmoid", kernel_initializer=init_glorot_uniform )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[StatefullMCC(np.linspace(0.45,0.55,11)) ])
 return model<split>
tf.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False,name='Adam' )
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_SEED ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in tqdm(enumerate(splits)) :
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit(train_X, train_y, batch_size=128, epochs=100, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape<compute_test_metric>
tf.keras.optimizers.Adamax(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Adamax' )
def threshold_search(y_true, y_proba):
thresholds = np.linspace(0.0,1.0,101)
 scores = [matthews_corrcoef(y_true,(y_proba > t ).astype(np.uint8)) for t in thresholds]
best_idx = np.argmax(scores)
 return thresholds[best_idx], scores[best_idx]<compute_test_metric>
def myLoss(y_true,y_pred):
sigma =(y_pred[:, 2] - y_pred[:, 0])
 delta = tf.abs(y_true[:, 0] - y_pred[:, 1])
 errore = tf.abs(sigma)- tf.abs(delta)
 return errore
best_threshold, best_score = threshold_search(y_val, preds_val)
 print(best_threshold, best_score )<load_from_csv>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 BEST = 0
def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma =(y_pred[:, 2] - y_pred[:, 0])
 fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric = -(( delta / sigma_clip)* sq2 + tf.math.log(sigma_clip * sq2)) /BEST
return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [qmin,qmed,qmax]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred

 v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda, err):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)+ err * myLoss(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( len(FE),), name="Patient")
 x = L.LeakyReLU(alpha=ALPHA )(z)
 x = L.Dense(HDO, kernel_regularizer=l2(L2))(x)
 x = L.LeakyReLU(alpha=ALPHA )(x)
 x = L.Dense(HDO, kernel_regularizer=l2(L2))(x)
 p1 = L.Dense(3, activation="swish", name="p1" )(x)
 model = M.Model(z, p1, name="CNN")
 model.compile(loss=mloss(use_metric, ERR), optimizer= OPTIMIZER, metrics=[score])
 return model
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()<predict_on_test>
print(len(tr['KGroup'].unique()))
preds_test = []
for i in tqdm(range(N_SPLITS)) :
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3)
<data_type_conversions>
a=tr['KGroup'].unique()
b=int(len(tr['KGroup'].unique())*(1-TESTSIZE))
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape<save_to_csv>
train_group = np.random.choice(a, b, replace=False)
 train_group = list(train_group)
 test_group = []
for k in range(1,len(tr['KGroup'].unique())+1):
if k not in train_group:
test_group.append(int(k))
print(len(train_group),len(test_group))
del a,b,k
submission['target'] = preds_test
submission.loc[submission["signal_id"].isin(exp_true), "target"] = 1
submission.loc[submission["signal_id"].isin(exp_false), "target"] = 0
submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
X_train = pd.DataFrame()
Y_train = pd.DataFrame()
for k in train_group:
X_train = X_train.append(tr.loc[tr.KGroup == k])
 Y_train = X_train['FVC']
del k
gc.enable() 
<set_options>
X_test = pd.DataFrame()
Y_test = pd.DataFrame()
if TESTSIZE > 0:
for k in test_group:
X_test = X_test.append(tr.loc[tr.KGroup == k])
 Y_test = X_test['FVC']
warnings.simplefilter(action='ignore', category=FutureWarning )<define_variables>
z = X_train[FE].values
z = np.reshape(z,(z.shape[0],z.shape[1],1))
y = Y_train.values
ze = sub[FE].values
ze = np.reshape(ze,(ze.shape[0],z.shape[1],1))
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
pred_act = np.zeros(( z.shape[0], 3))
if TESTSIZE > 0:
z_test = X_test[FE].values
z_test = np.reshape(z_test,(z_test.shape[0],z_test.shape[1],1))
y_test = Y_test.values
pred_test = np.zeros(( z_test.shape[0], 3))
N_SPLITS = 5
sample_size = 800000<compute_test_metric>
def analyze_model(trax, tray, testx, testy):
a = net.evaluate(trax,tray, verbose=0, batch_size=BATCH_SIZE)
 b = net.evaluate(testx,testy, verbose=0, batch_size=BATCH_SIZE)
 c = tf.constant(BEST, dtype='float32')
 print("train:", a*c,'
')
 print("test:", b*c,'
' )
def matthews_correlation(y_true, y_pred):

 y_true = K.cast(y_true, float)
 y_pred = K.cast(y_pred, float)
 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )<choose_model_class>
def analyze_GROUP_model(a,b,c):
print("train:", a[0] / NFOLD ,a[1] * BEST / NFOLD,'
')
 print("val:", b[0] / NFOLD ,b[1] * BEST / NFOLD,'
')
 print("test:", c[0] / NFOLD ,c[1] * BEST / NFOLD,'
' )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<load_from_csv>
def laplace_Group_model(scope,base_x,base_y, predicted, bias):
actualFVC = base_y
predictedFVC = predicted
sigma_opt_actual = mean_absolute_error(actualFVC, predictedFVC[:, 1])
 sigmaSTD = mean_squared_error(actualFVC, predictedFVC[:, 1])
 unc_actual =(predictedFVC[:,2] - predictedFVC[:, 0])* bias
sigma_mean_actual = np.mean(unc_actual)
 best = laplace_log_likelihood(actual_fvc = actualFVC, predicted_fvc = actualFVC,\
confidence= 0)
 print('laplace_log_likelihood su '+ scope +':
')
 print("opt_actual", laplace_log_likelihood(actual_fvc = actualFVC, predicted_fvc = predictedFVC[:,1],\
confidence= sigma_opt_actual))
print("mean_squared", laplace_log_likelihood(actual_fvc = actualFVC, predicted_fvc = predictedFVC[:,1],\
confidence= sigmaSTD))
print("unc_actual", laplace_log_likelihood(actual_fvc = actualFVC, predicted_fvc = predictedFVC[:,1],\
confidence= unc_actual))
print("mean_actual", laplace_log_likelihood(actual_fvc = actualFVC, predicted_fvc = predictedFVC[:,1],\
confidence= sigma_mean_actual))
return(best )
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()<define_variables>
def laplace_model(scope,base_x,base_y,base_pred):
actualFVC = base_y
predictedFVC = base_pred
sigma_opt_actual = mean_absolute_error(actualFVC, predictedFVC[:, 1])
 sigmaSTD = mean_squared_error(actualFVC, predictedFVC[:, 1])
 unc_actual = predictedFVC[:,2] - predictedFVC[:, 0]
sigma_mean_actual = np.mean(unc_actual)
 best = laplace_log_likelihood(actual_fvc = actualFVC, predicted_fvc = actualFVC,\
confidence= 0)
 print('laplace_log_likelihood su '+ scope +':
')
 print("opt_actual", laplace_log_likelihood(actual_fvc = actualFVC, predicted_fvc = predictedFVC[:,1],\
confidence= sigma_opt_actual))
print("mean_squared", laplace_log_likelihood(actual_fvc = actualFVC, predicted_fvc = predictedFVC[:,1],\
confidence= sigmaSTD))
print("unc_actual", laplace_log_likelihood(actual_fvc = actualFVC, predicted_fvc = predictedFVC[:,1],\
confidence= unc_actual))
print("mean_actual", laplace_log_likelihood(actual_fvc = actualFVC, predicted_fvc = predictedFVC[:,1],\
confidence= sigma_mean_actual))
return(best )
max_num = 127
min_num = -128<categorify>
BEST = laplace_model('BEST',z,y,pred)
 start_at = time()
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]<prepare_x_and_y>
%%time
if GROUPS:
history=History()
groups = X_train.KGroup
gkf = GroupKFold(n_splits= NFOLD)
 cnt=0
net_train = np.zeros(2,)
 net_val = np.zeros(2,)
 net_test = np.zeros(2,)
 start_at = time()
for tr_idx, val_idx in gkf.split(z, y, groups=groups):
cnt += 1
print(f"GROUP {cnt}")
 net = make_model()
start_Fold = time()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0, callbacks=[history])
 exec_time = time() - start_Fold
a = net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE)
 b = net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE)
 print("train su fold:", a[0],a[1]*BEST)
 print("val su fold:", b[0],b[1]*BEST)
 net_train += a
net_val += b
if TESTSIZE > 0:
c = net.evaluate(z_test, y_test, verbose=0, batch_size=BATCH_SIZE)
 print("test su fold:", c[0],c[1]*BEST,)
 net_test += c
pred_test += net.predict(z_test, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
pred[val_idx] += net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
print("Tempo di addestramento: %d minuti e %d secondi" %(exec_time/60, exec_time%60),'
')
 exec_time = time() - start_at
print("
Tempo totale di addestramento: %d minuti e %d secondi" %(exec_time/60, exec_time%60),'
')
 del a,b
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<categorify>
if GROUPS == False:
print("Inizio addestramento:...
 ")
 history = History()
net = make_model()
net.fit(z, y, batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_split=VAL_SPLIT, verbose=0, callbacks=[history])
 exec_time = time() - start_at
print("Tempo totale di addestramento: %d minuti e %d secondi" %(exec_time/60, exec_time%60),'
')
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X_aug, y_aug = augment(X_temp,y_temp,t=2)
 X.append(X_aug)
 y.append(y_aug)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )<normalization>
if GROUPS:
analyze_GROUP_model(net_train, net_val, net_test)
 else:
if TESTSIZE > 0:
analyze_model(z,y,z_test, y_test)
 else:
analyze_model(z,y,z, y )
def petrosian_fd(x):

 n = len(x)
 diff = np.ediff1d(x)
 N_delta =(diff[1:-1] * diff[0:-2] < 0 ).sum()
return np.log10(n)/(np.log10(n)+ np.log10(n /(n + 0.4 * N_delta)))
 def katz_fd(x):

 x = np.array(x)
 dists = np.abs(np.ediff1d(x))
ll = dists.sum()
ln = np.log10(np.divide(ll, dists.mean()))
aux_d = x - x[0]
d = np.max(np.abs(aux_d[1:]))
return np.divide(ln, np.add(ln, np.log10(np.divide(d, ll))))
@jit('UniTuple(float64, 2 )(float64[:], float64[:])', nopython=True)
 def _linear_regression(x, y):

 n_times = x.size
sx2 = 0
sx = 0
sy = 0
sxy = 0
for j in range(n_times):
sx2 += x[j] ** 2
sx += x[j]
sxy += x[j] * y[j]
sy += y[j]
den = n_times * sx2 -(sx ** 2)
 num = n_times * sxy - sx * sy
slope = num / den
intercept = np.mean(y)- slope * np.mean(x)
 return slope, intercept
@jit('i8[:](f8, f8, f8)', nopython=True)
 def _log_n(min_n, max_n, factor):

 max_i = int(floor(log(1.0 * max_n / min_n)/ log(factor)))
 ns = [min_n]
for i in range(max_i + 1):
n = int(floor(min_n *(factor ** i)))
 if n > ns[-1]:
ns.append(n)
 return np.array(ns, dtype=np.int64)
 @jit('float64(float64[:], int32)')
 def _higuchi_fd(x, kmax):

 n_times = x.size
lk = np.empty(kmax)
 x_reg = np.empty(kmax)
 y_reg = np.empty(kmax)
 for k in range(1, kmax + 1):
lm = np.empty(( k,))
for m in range(k):
ll = 0
n_max = floor(( n_times - m - 1)/ k)
 n_max = int(n_max)
 for j in range(1, n_max):
ll += abs(x[m + j * k] - x[m +(j - 1)* k])
 ll /= k
ll *=(n_times - 1)/(k * n_max)
 lm[m] = ll
m_lm = 0
for m in range(k):
m_lm += lm[m]
m_lm /= k
lk[k - 1] = m_lm
x_reg[k - 1] = log(1./ k)
 y_reg[k - 1] = log(m_lm)
 higuchi, _ = _linear_regression(x_reg, y_reg)
 return higuchi
def higuchi_fd(x, kmax=10):

 x = np.asarray(x, dtype=np.float64)
 kmax = int(kmax)
 return _higuchi_fd(x, kmax )<categorify>
if GROUPS:
for BIAS in(1,0,0.2,0.4,0.6,0.8,1.0,1.2,1.4,1.6,1.8,2):
print('
BIAS:',BIAS,'
')
 laplace_Group_model('training',z,y,pred, BIAS)
 print('
x--------x
')
 if TESTSIZE > 0: laplace_Group_model('test',z_test,y_test,pred_test, BIAS)
 else:
laplace_model('training',z,y,pred)
 print('
x--------x
')
 if TESTSIZE > 0: laplace_model('test',z_test,y_test,pred_test )
def _embed(x, order=3, delay=1):

 N = len(x)
 if order * delay > N:
raise ValueError("Error: order * delay should be lower than x.size")
 if delay < 1:
raise ValueError("Delay has to be at least 1.")
 if order < 2:
raise ValueError("Order has to be at least 2.")
 Y = np.zeros(( order, N -(order - 1)* delay))
for i in range(order):
Y[i] = x[i * delay:i * delay + Y.shape[1]]
return Y.T
all = ['perm_entropy', 'spectral_entropy', 'svd_entropy', 'app_entropy',
'sample_entropy']
def perm_entropy(x, order=3, delay=1, normalize=False):

 x = np.array(x)
 ran_order = range(order)
 hashmult = np.power(order, ran_order)
 sorted_idx = _embed(x, order=order, delay=delay ).argsort(kind='quicksort')
 hashval =(np.multiply(sorted_idx, hashmult)).sum(1)
 _, c = np.unique(hashval, return_counts=True)
 p = np.true_divide(c, c.sum())
 pe = -np.multiply(p, np.log2(p)).sum()
if normalize:
pe /= np.log2(factorial(order))
return pe
def spectral_entropy(x, sf, method='fft', nperseg=None, normalize=False):

 x = np.array(x)
 if method == 'fft':
_, psd = periodogram(x, sf)
 elif method == 'welch':
_, psd = welch(x, sf, nperseg=nperseg)
 psd_norm = np.divide(psd, psd.sum())
 se = -np.multiply(psd_norm, np.log2(psd_norm)).sum()
if normalize:
se /= np.log2(psd_norm.size)
 return se
def svd_entropy(x, order=3, delay=1, normalize=False):

 x = np.array(x)
 mat = _embed(x, order=order, delay=delay)
 W = np.linalg.svd(mat, compute_uv=False)
 W /= sum(W)
 svd_e = -np.multiply(W, np.log2(W)).sum()
if normalize:
svd_e /= np.log2(order)
 return svd_e
def _app_samp_entropy(x, order, metric='chebyshev', approximate=True):

 _all_metrics = KDTree.valid_metrics
if metric not in _all_metrics:
raise ValueError('The given metric(%s)is not valid.The valid '
'metric names are: %s' %(metric, _all_metrics))
phi = np.zeros(2)
 r = 0.2 * np.std(x, axis=-1, ddof=1)
 _emb_data1 = _embed(x, order, 1)
 if approximate:
emb_data1 = _emb_data1
else:
emb_data1 = _emb_data1[:-1]
count1 = KDTree(emb_data1, metric=metric ).query_radius(emb_data1, r,
count_only=True
).astype(np.float64)
 emb_data2 = _embed(x, order + 1, 1)
 count2 = KDTree(emb_data2, metric=metric ).query_radius(emb_data2, r,
count_only=True
).astype(np.float64)
 if approximate:
phi[0] = np.mean(np.log(count1 / emb_data1.shape[0]))
phi[1] = np.mean(np.log(count2 / emb_data2.shape[0]))
else:
phi[0] = np.mean(( count1 - 1)/(emb_data1.shape[0] - 1))
phi[1] = np.mean(( count2 - 1)/(emb_data2.shape[0] - 1))
return phi
@jit('f8(f8[:], i4, f8)', nopython=True)
 def _numba_sampen(x, mm=2, r=0.2):

 n = x.size
n1 = n - 1
mm += 1
mm_dbld = 2 * mm
r *= x.std()
run = [0] * n
run1 = run[:]
r1 = [0] *(n * mm_dbld)
 a = [0] * mm
b = a[:]
p = a[:]
for i in range(n1):
nj = n1 - i
for jj in range(nj):
j = jj + i + 1
if abs(x[j] - x[i])< r:
run[jj] = run1[jj] + 1
m1 = mm if mm < run[jj] else run[jj]
for m in range(m1):
a[m] += 1
if j < n1:
b[m] += 1
else:
run[jj] = 0
for j in range(mm_dbld):
run1[j] = run[j]
r1[i + n * j] = run[j]
if nj > mm_dbld - 1:
for j in range(mm_dbld, nj):
run1[j] = run[j]
m = mm - 1
while m > 0:
b[m] = b[m - 1]
m -= 1
b[0] = n * n1 / 2
a = np.array([float(aa)for aa in a])
 b = np.array([float(bb)for bb in b])
 p = np.true_divide(a, b)
 return -log(p[-1])
 def app_entropy(x, order=2, metric='chebyshev'):

 phi = _app_samp_entropy(x, order=order, metric=metric, approximate=True)
 return np.subtract(phi[0], phi[1])
 def sample_entropy(x, order=2, metric='chebyshev'):

 x = np.asarray(x, dtype=np.float64)
 if metric == 'chebyshev' and x.size < 5000:
return _numba_sampen(x, mm=order, r=0.2)
 else:
phi = _app_samp_entropy(x, order=order, metric=metric,
approximate=False)
 return -np.log(np.divide(phi[1], phi[0]))<compute_test_metric>
predxs = pred
idxs = np.random.randint(0, y.shape[0], 100)
 idxs = np.sort(idxs )
def entropy_and_fractal_dim(x):
return [perm_entropy(x), svd_entropy(x), app_entropy(x), sample_entropy(x), petrosian_fd(x), katz_fd(x), higuchi_fd(x),scipy.stats.kurtosis(x),denoised_std(x),signal_entropy(x),detail_coeffs_entropy(x, wavelet='db1')]+list(peaks(x))<categorify>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print('sigma_opt mean_absolute_error(y, pred[:, 1]):',sigma_opt,
'
sigma_mean np.mean(unc)unc = pred[:,2] - pred[:, 0]:',sigma_mean )
signals = X.reshape(( len(X), X.shape[1]*X.shape[2]))
features = []
for signal in signals:
features.append(entropy_and_fractal_dim(signal))<normalization>
CORRECTION = tf.constant(1.0, dtype='float32' )
features = np.array(features ).reshape(( len(features), 18))
scaler = MinMaxScaler(feature_range=(0, 1))
scaler.fit(features)
 features = scaler.transform(features )<choose_model_class>
sab = sub.copy()
def model_lstm(input_shape, feat_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
feat = Input(shape=(feat_shape[1],))
bi_lstm = Bidirectional(CuDNNLSTM(128, return_sequences=True), merge_mode='concat' )(inp)
 x = Dropout(0.23 )(bi_lstm)
 bi_gru = Bidirectional(CuDNNGRU(64, return_sequences=True), merge_mode='concat' )(x)
 attention = Attention(input_shape[1] )(bi_gru)
 x = concatenate([attention, feat], axis=1)
 x = Dropout(0.2 )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[inp, feat], outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model<split>
sab['FVC1'] = pe[:, 1]
sab['Confidence1'] =(pe[:, 2] - pe[:, 0] )
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_feat, train_y, val_X, val_feat, val_y = X[train_idx], features[train_idx], y[train_idx], X[val_idx], features[val_idx], y[val_idx]
model = model_lstm(train_X.shape, features.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit([train_X, train_feat], train_y, batch_size=128, epochs=70, validation_data=([val_X, val_feat], val_y), callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict([val_X, val_feat], batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape 
<find_best_params>
subm = sab[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = abs(subm.loc[~subm.FVC1.isnull() ,'Confidence1'])
 subm.describe().T
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result<compute_test_metric>
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
subm.describe().T
optimal_values = threshold_search(y_val, preds_val)
 best_threshold = optimal_values['threshold']
best_score = optimal_values['matthews_correlation']<compute_test_metric>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
print("Optimal Threshold : " + str(best_threshold))
print("Best Matthews Correlation : " + str(best_score))<load_from_csv>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False)
 print('Saved',GROUPS )
<drop_column><EOS>