Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<categorify>
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import scipy
from tqdm.notebook import tqdm
class RNNEncoderBase(nn.Module):
def __init__(self, config, modules, in_size):
super().__init__()
rnns = []
input_size = in_size
for module in modules:
rnn = module(
 input_size=input_size,
hidden_size=config.encoder_n_hidden,
bidirectional=config.encoder_bidirectional,
batch_first=True, 
)
 n_direction = int(config.encoder_bidirectional)+ 1
input_size = n_direction * config.encoder_n_hidden
rnns.append(rnn)
 self.rnns = nn.ModuleList(rnns)
 self.out_size = n_direction * config.encoder_n_hidden
@classmethod
def add_args(self, parser):
parser.add_argument('--encoder-bidirectional', type=bool, default=True)
 parser.add_argument('--encoder-dropout', type=float, default=0.)
parser.add_argument('--encoder-n-hidden', type=int)
 parser.add_argument('--encoder-n-layers', type=int)
 parser.add_argument('--encoder-aggregator', type=str,
choices=AGGREGATOR_REGISTRY)
 def forward(self, input, mask):
h = input
for rnn in self.rnns:
h, _ = rnn(h)
 return h
@register_encoder('lstm')
 class LSTMEncoder(RNNEncoderBase):
def __init__(self, config, in_size):
modules = [nn.LSTM] * config.encoder_n_layers
super().__init__(config, modules, in_size)
 @register_encoder('gru')
 class GRUEncoder(RNNEncoderBase):
def __init__(self, config, in_size):
assert config.encoder_n_layers > 1
modules = [nn.GRU] * config.encoder_n_layers
super().__init__(config, modules, in_size)
 @register_encoder('lstmgru')
 class LSTMGRUEncoder(RNNEncoderBase):
def __init__(self, config, in_size):
assert config.encoder_n_layers > 1
modules = [nn.LSTM] *(config.encoder_n_layers - 1)+ [nn.GRU]
super().__init__(config, modules, in_size)
 @register_encoder('grulstm')
 class GRULSTMEncoder(RNNEncoderBase):
def __init__(self, config, in_size):
assert config.encoder_n_layers > 1
modules = [nn.GRU] *(config.encoder_n_layers - 1)+ [nn.LSTM]
super().__init__(config, modules, in_size )<find_best_model_class>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 train.head()
class BaseEnsembler(metaclass=ABCMeta):
def __init__(self, config, models, results):
super().__init__()
self.config = config
self.models = models
self.results = results
@abstractmethod
def fit(self, X, t, test_size=0.1):
pass
@abstractmethod
def predict_proba(self, X, X2):
pass
def predict(self, X, X2):
y = self.predict_proba(X, X2)
 return(y > self.threshold ).astype('i')
 class AverageEnsembler(BaseEnsembler):
def __init__(self, config, models, results):
self.config = config
self.models = models
self.results = results
self.device = config.device
self.batchsize_train = config.batchsize
self.batchsize_valid = config.batchsize_valid
self.threshold_cv = np.array(
 [m.threshold for m in models] ).mean()
self.threshold = self.threshold_cv
def fit(self, X, X2, t, test_size=0.1):
pass
def predict_proba(self, X, X2):
pred_X = X.to(self.device)
 pred_X2 = X2.to(self.device)
 dataset = torch.utils.data.TensorDataset(pred_X, pred_X2)
 iterator = DataLoader(
 dataset, batch_size=self.batchsize_valid, shuffle=False)
 ys = defaultdict(list)
 for batch in tqdm(iterator, desc='submit', leave=False):
for i, model in enumerate(self.models):
model.eval()
ys[i].append(model.predict_proba(*batch))
ys = np.concatenate(
 [np.concatenate(_ys)for _ys in ys.values() ], axis=1)
 y = ys.mean(axis=1, keepdims=True)
 return y 
<compute_train_metric>
model_stats = ols('FVC ~ Weeks + Percent + Age + Sex + SmokingStatus',
data=train ).fit()
print(model_stats.summary() )
class BinaryClassifier(nn.Module):
default_config = None
def __init__(self, embedding, encoder, aggregator, mlp, out, lossfunc):
super().__init__()
self.embedding = embedding
self.encoder = encoder
self.aggregator = aggregator
self.mlp = mlp
self.out = out
self.lossfunc = lossfunc
def calc_loss(self, X, X2, t, W=None):
y = self.forward(X, X2)
 loss = self.lossfunc(y, t)
 output = dict(
 y=torch.sigmoid(y ).cpu().detach().numpy() ,
t=t.cpu().detach().numpy() ,
loss=loss.cpu().detach().numpy() , 
)
 return loss, output
def to_device(self, device):
self.device = device
self.to(device)
 return self
def forward(self, X, X2):
h = self.predict_features(X, X2)
 out = self.out(h)
 return out
def predict_proba(self, X, X2):
y = self.forward(X, X2)
 proba = torch.sigmoid(y ).cpu().detach().numpy()
return proba
def predict_features(self, X, X2):
mask = X != 0
maxlen =(mask == 1 ).any(dim=0 ).sum()
X = X[:, :maxlen]
mask = mask[:, :maxlen]
h = self.embedding(X)
 h = self.encoder(h, mask)
 h = self.aggregator(h, mask)
 h = self.mlp(h, X2)
 return h<load_pretrained>
train_org = train.copy()
class NNModuleWrapperBase(nn.Module, metaclass=ABCMeta):
@abstractmethod
def add_args(cls, parser):
raise NotImplementedError()
@abstractmethod
def add_extra_args(cls, parser):
raise NotImplementedError()
class EmbeddingWrapper(NNModuleWrapperBase):
default_config = None
def __init__(self, config, embedding_matrix):
super().__init__()
self.config = config
self.module = nn.Embedding.from_pretrained(
 torch.Tensor(embedding_matrix), freeze=True)
 if self.config.embedding_dropout1d > 0:
self.dropout1d = nn.Dropout(config.embedding_dropout1d)
 if self.config.embedding_dropout2d > 0:
self.dropout2d = nn.Dropout2d(config.embedding_dropout2d)
 if self.config.embedding_spatial_dropout > 0:
self.spatial_dropout = nn.Dropout2d(
 config.embedding_spatial_dropout)
 self.out_size = embedding_matrix.shape[1]
@classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument('--embedding-dropout1d', type=float, default=0.)
parser.add_argument('--embedding-dropout2d', type=float, default=0.)
parser.add_argument('--embedding-spatial-dropout',
type=float, default=0.)
parser.set_defaults(**cls.default_config)
 @classmethod
def add_extra_args(cls, parser, config):
pass
def forward(self, X):
h = self.module(X)
 if self.config.embedding_dropout1d > 0:
h = self.dropout1d(h)
 if self.config.embedding_dropout2d > 0:
h = self.dropout2d(h)
 if self.config.embedding_spatial_dropout > 0:
h = h.permute(0, 2, 1)
 h = self.spatial_dropout(h)
 h = h.permute(0, 2, 1)
 return h
class EncoderWrapper(nn.Module):
registry = ENCODER_REGISTRY
def __init__(self, config, in_size):
super().__init__()
self.config = config
self.module = self.registry[config.encoder](config, in_size)
 self.out_size = self.module.out_size
@classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument(
 '--encoder', choices=cls.registry)
 parser.set_defaults(**cls.default_config)
 @classmethod
def add_extra_args(cls, parser, config):
assert isinstance(cls.default_extra_config, dict)
 cls.registry[config.encoder].add_args(parser)
 parser.set_defaults(**cls.default_extra_config)
 def forward(self, X, mask):
h = self.module(X, mask)
 return h
class AggregatorWrapper(NNModuleWrapperBase):
default_config = None
registry = AGGREGATOR_REGISTRY
def __init__(self, config):
super().__init__()
self.config = config
self.module = self.registry[config.aggregator]()
@classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument('--aggregator',
choices=cls.registry)
 parser.set_defaults(**cls.default_config)
 @classmethod
def add_extra_args(cls, parser, config):
pass
def forward(self, X, mask):
h = self.module(X, mask)
 return h
class MLPWrapper(NNModuleWrapperBase):
default_config = None
def __init__(self, config, in_size):
super().__init__()
self.in_size = in_size
self.config = config
assert isinstance(config.mlp_n_hiddens, list)
 layers = []
if config.mlp_bn0:
layers.append(nn.BatchNorm1d(in_size))
if config.mlp_dropout0 > 0:
layers.append(nn.Dropout(config.mlp_dropout0))
for n_hidden in config.mlp_n_hiddens:
layers.append(nn.Linear(in_size, n_hidden))
if config.mlp_actfun is not None:
layers.append(config.mlp_actfun)
 if config.mlp_bn:
layers.append(nn.BatchNorm1d(n_hidden))
if config.mlp_dropout > 0:
layers.append(nn.Dropout(config.mlp_dropout))
in_size = n_hidden
self.layers = nn.Sequential(*layers)
 @classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument('--mlp-n-hiddens', type=list)
 parser.add_argument('--mlp-bn', type=bool)
 parser.add_argument('--mlp-bn0', type=bool)
 parser.add_argument('--mlp-dropout', type=float, default=0.)
parser.add_argument('--mlp-dropout0', type=float, default=0.)
parser.add_argument('--mlp-actfun', default=0.)
parser.set_defaults(**cls.default_config)
 @classmethod
def add_extra_args(cls, parser, config):
pass
def forward(self, X, X2):
h = X
if X.shape[1] + X2.shape[1] == self.in_size:
h = torch.cat([h, X2], dim=1)
 h = self.layers(h)
 return h 
<set_options>
weeks_to_pred = np.array([i for i in range(-12, 134)] ).astype(np.float32 )
class ExperimentConfigBuilderPresets(ExperimentConfigBuilderBase):
default_config = dict(
 maxlen=72,
vocab_mincount=5,
scale_batchsize=[],
validate_from=2, 
)
 class TextNormalizerPresets(TextNormalizerWrapper):
default_config = dict(
 normalizers=[
'lower',
'misspell',
'punct',
'number+underscore'
] 
)
 class TextTokenizerPresets(TextTokenizerWrapper):
default_config = dict(
 tokenizer='space' 
)
 class WordEmbeddingFeaturizerPresets(WordEmbeddingFeaturizerWrapper):
default_config = dict(
 use_pretrained_vectors=['glove', 'paragram'],
word_embedding_features=['pretrained', 'word2vec'], 
)
 default_extra_config = dict(
 finetune_word2vec_init_unk='zeros',
finetune_word2vec_mincount=1,
finetune_word2vec_workers=1,
finetune_word2vec_iter=5,
finetune_word2vec_size=300,
finetune_word2vec_sg=0,
finetune_word2vec_sorted_vocab=0, 
)
 class WordExtraFeaturizerPresets(WordExtraFeaturizerWrapper):
default_config = dict(
 word_extra_features=[], 
)
 class SentenceExtraFeaturizerPresets(SentenceExtraFeaturizerWrapper):
default_config = dict(
 sentence_extra_features=[], 
)
 class PreprocessorPresets(WordbasedPreprocessor):
def build_word_features(self, word_embedding_featurizer,
embedding_matrices, word_extra_features):
embedding = np.stack(list(embedding_matrices.values()))
unk =(embedding[0] == 0 ).all(axis=1)
 mean, std = embedding[0, ~unk].mean() , embedding[0, ~unk].std()
unk_and_hfq = unk & word_embedding_featurizer.vocab.hfq
noise = np.random.normal(
 mean, std,(unk_and_hfq.sum() , embedding[0].shape[1]))
embedding[0, unk_and_hfq] = noise
embedding[0, 0] = 0
embedding = embedding.mean(axis=0)
 word_features = np.concatenate(
 [embedding, word_extra_features], axis=1)
 return word_features
class EmbeddingPresets(EmbeddingWrapper):
default_config = dict(
 embedding_dropout1d=0.2, 
)
 class EncoderPresets(EncoderWrapper):
default_config = dict(
 encoder='lstm', 
)
 default_extra_config = dict(
 encoder_bidirectional=True,
encoder_dropout=0.,
encoder_n_layers=2,
encoder_n_hidden=128, 
)
 class AggregatorPresets(AggregatorWrapper):
default_config = dict(
 aggregator='max', 
)
 class MLPPresets(MLPWrapper):
default_config = dict(
 mlp_n_hiddens=[128, 128],
mlp_bn0=False,
mlp_dropout0=0.,
mlp_bn=True,
mlp_actfun=nn.ReLU(True), 
)
 class EnsemblerPresets(AverageEnsembler):
pass 
<compute_train_metric>
A_list = []
for i in list(A.values()):
if i > 0:
val = i -(np.max(list(A.values())) + 0.00000001)
 else:
val = i
A_list.append(val)
 x=0
for i in A.keys() :
A[i] = A_list[x]
x += 1
def classification_metrics(ys, ts):
scores = {}
if len(np.unique(ts)) > 1:
precs, recs, thresholds = metrics.precision_recall_curve(ts, ys)
 thresholds = np.append(thresholds, 1.001)
 idx =(precs != 0)*(recs != 0)
 precs, recs, thresholds = precs[idx], recs[idx], thresholds[idx]
fbetas = 2 /(1 / precs + 1 / recs)
 best_idx = np.argmax(fbetas)
 threshold = thresholds[best_idx]
prec = precs[best_idx]
rec = recs[best_idx]
fbeta = fbetas[best_idx]
scores['ap'] = metrics.average_precision_score(ts, ys)
 scores['rocauc'] = metrics.roc_auc_score(ts, ys)
 scores['threshold'] = threshold
scores['prec'] = prec
scores['rec'] = rec
scores['fbeta'] = fbeta
return scores
class ClassificationResult(object):
def __init__(self, name, outdir=None, postfix=None, main_metrics='fbeta'):
self.initialize()
self.name = name
self.postfix = postfix
self.outdir = outdir
self.summary = None
self.main_metrics = main_metrics
self.n_trained = 0
def initialize(self):
self.losses = []
self.ys = []
self.ts = []
def add_record(self, loss, y, t):
self.losses.append(loss)
 self.ys.append(y)
 self.ts.append(t)
 self.n_trained += len(y)
 def calc_score(self, epoch):
loss = np.array(self.losses ).mean()
self.ys, self.ts = np.concatenate(self.ys), np.concatenate(self.ts)
 score = classification_metrics(self.ys, self.ts)
 summary = dict(name=self.name, loss=loss, **score)
 if len(score)> 0:
if self.summary is None:
self.summary = pd.DataFrame([summary], index=[epoch])
 self.summary.index.name = 'epoch'
else:
self.summary.loc[epoch] = summary
if self.best_epoch == epoch:
self.best_ys = self.ys
self.best_ts = self.ts
self.initialize()
def get_dict(self):
loss, fbeta, epoch = 0, 0, 0
if self.summary is not None:
row = self.summary.iloc[-1]
epoch = row.name
loss = row.loss
fbeta = row.fbeta
return {
'epoch': epoch,
'loss': loss,
'fbeta': fbeta,
}
@property
def fbeta(self):
if self.summary is None:
return 0
else:
return self.summary.fbeta[-1]
@property
def best_fbeta(self):
return self.summary[self.main_metrics].max()
@property
def best_epoch(self):
return self.summary[self.main_metrics].idxmax()
@property
def best_threshold(self):
idx = self.summary[self.main_metrics].idxmax()
return self.summary['threshold'][idx]<categorify>
def get_img(path):
d = pydicom.dcmread(path)
 return cv2.resize(( d.pixel_array - d.RescaleIntercept)/(d.RescaleSlope * 1000),(512, 512))
%%cython
c
cdef class ApplyNdArray:
cdef func
cdef dtype
cdef dims
cdef int processes
def __init__(self, func, processes=1, dtype=object, dims=None):
self.func = func
self.processes = processes
self.dtype = dtype
self.dims = dims
def __call__(self, arr):
if self.processes == 1:
return self.apply(arr)
 else:
return self.apply_parallel(arr)
 cpdef apply(self, arr):
cdef int i
cdef int n = len(arr)
 if self.dims is not None:
shape =(n, *self.dims)
 else:
shape = n
cdef res = np.empty(shape, dtype=self.dtype)
 for i in range(n):
res[i] = self.func(arr[i])
 return res
cpdef apply_parallel(self, arr):
cdef list arrs = np.array_split(arr, self.processes)
 with Pool(processes=self.processes)as pool:
outputs = pool.map(self.apply, arrs)
 return np.concatenate(outputs, axis=0)
<normalization>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.unique() :
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 numb = [float(i[:-4])for i in ldir]
self.train_data[p] = [i for i in os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15]
def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 mask = cv2.resize(cv2.imread(f'.. /input/osic-pulmonary-fibrosis-progression-lungs-mask/mask_clear/mask_clear/{k}/{i[:-4]}.jpg', 0),(512, 512)) > 0
img[~mask] = 0
x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
def load_module(filename):
assert isinstance(filename, Path)
 name = filename.stem
spec = importlib.util.spec_from_file_location(name, filename)
 mod = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(mod)
 sys.modules[mod.__name__] = mod
return mod
def rmtree_after_confirmation(path, force=False):
if Path(path ).exists() :
if not force and not prompter.yesno('Overwrite %s?' % path):
sys.exit(0)
 else:
shutil.rmtree(path)
 def pad_sequence(xs, length, padding_value=0):
assert isinstance(xs, list)
 n_padding = length - len(xs)
 return np.array(xs + [padding_value] * n_padding, 'i')[:length]
def set_seed(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
class Pipeline(object):
def __init__(self, *modules):
self.modules = modules
def __call__(self, x):
for module in self.modules:
x = module(x)
 return x<split>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_exp(beta=0.025):
def exp_coef(x, beta=beta):
return 0.5 * K.tanh(x*beta)- beta
return exp_coef
def get_model(shape=(512, 512, 1)) :
def res_block(x, n_features):
_x = x
x = BatchNormalization()(x)
 x = LeakyReLU(0.05 )(x)
 x = Conv2D(n_features, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 x = Add()([_x, x])
 return x
inp = Input(shape=shape)
 x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same' )(inp)
 x = BatchNormalization()(x)
 x = LeakyReLU(0.05 )(x)
 x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 x = BatchNormalization()(x)
 x = LeakyReLU(0.05 )(x)
 x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
 x = Conv2D(8, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 for _ in range(2):
x = res_block(x, 8)
 x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
 x = Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 for _ in range(2):
x = res_block(x, 16)
 x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
 x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 for _ in range(3):
x = res_block(x, 32)
 x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
 x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 for _ in range(3):
x = res_block(x, 64)
 x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
 x = Conv2D(128, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 for _ in range(3):
x = res_block(x, 128)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(0.5 )(x)
 x = Dense(1, activation='tanh' )(x)
 x = Dense(1, activation=get_exp(beta=0.025))(x)
 return Model([inp, inp2] , x )
%%time
exec(modules)
 config = ExperimentConfigBuilder().build(args=[])
 print(config)
 start = time.time()
set_seed(config.seed)
 train_df, submit_df = load_qiqc(n_rows=config.n_rows)
 datasets = build_datasets(train_df, submit_df, config.holdout, config.seed)
 train_dataset, test_dataset, submit_dataset = datasets<string_transform>
def exp_coef(x, beta=1.0):
orig = x
return 0.5 * K.tanh(x*beta)- beta
a = [i / 10 for i in range(-10, 10)]
plt.plot(a, exp_coef(tf.convert_to_tensor(np.array(a ).astype(np.float32)) , beta=0.025), 'r.' )
%%time
print('Tokenize texts...')
 preprocessor = Preprocessor()
normalizer = TextNormalizer(config)
 tokenizer = TextTokenizer(config)
 train_dataset.tokens, test_dataset.tokens, submit_dataset.tokens = \
preprocessor.tokenize(datasets, normalizer, tokenizer )<define_variables>
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001), loss='mse', metrics=['mean_absolute_error'] )
%%time
print('Build vocabulary...')
 vocab = preprocessor.build_vocab(datasets, config )<categorify>
tr_p, vl_p = train_test_split(P,
shuffle=True,
train_size= 0.8 )
%%time
print('Build token ids...')
 train_dataset.tids, test_dataset.tids, submit_dataset.tids = \
preprocessor.build_tokenids(datasets, vocab, config )<feature_engineering>
er = tf.keras.callbacks.EarlyStopping(
 monitor="val_loss",
min_delta=1e-6,
patience=10,
verbose=0,
mode="auto",
baseline=None,
restore_best_weights=True,
)
%%time
print('Build sentence extra features...')
 sentence_extra_featurizer = SentenceExtraFeaturizer(config)
 train_dataset._X2, test_dataset._X2, submit_dataset._X2 = \
preprocessor.build_sentence_features(
 datasets, sentence_extra_featurizer)
 [d.build(config.device)for d in datasets]<load_pretrained>
model.fit_generator(IGenerator(keys=tr_p,
a = A,
tab = TAB),
steps_per_epoch = 100,
validation_data=IGenerator(keys=vl_p,
a = A,
tab = TAB),
validation_steps = 20,
callbacks = [er],
epochs=30 )
%%time
print('Load pretrained vectors...')
 pretrained_vectors = load_pretrained_vectors(
 config.use_pretrained_vectors, vocab.token2id, test=config.test )<categorify>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
%%time
print('Build word embedding matrix...')
 word_embedding_featurizer = WordEmbeddingFeaturizer(config, vocab)
 embedding_matrices = preprocessor.build_embedding_matrices(
 datasets, word_embedding_featurizer, vocab, pretrained_vectors )<categorify>
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = _a
a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = np.exp(a*(weeks_true - weeks_true[0])+ np.log(fvc_true[0]))
sigma = percent_true[0] - np.exp(abs(weeks_true - weeks_true[0])* a)
 m.append(score(fvc_true, fvc, sigma))
print(np.mean(m))
metric.append(np.mean(m))
%%time
print('Build word extra features...')
 word_extra_featurizer = WordExtraFeaturizer(config, vocab)
 word_extra_features = word_extra_featurizer(vocab )<load_pretrained>
q =(np.argmin(metric)+ 1)/ 10
q
%%time
print('Build models...')
 word_features_cv = [
preprocessor.build_word_features(
 word_embedding_featurizer, embedding_matrices, word_extra_features)
 for i in range(config.cv)]
models = [
build_model(
 config, word_features, sentence_extra_featurizer.n_dims 
)for word_features in word_features_cv]<find_best_params>
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 sub.head()
print(models[0] )<split>
test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv' )
%%time
print('Start training...')
 splitter = sklearn.model_selection.StratifiedKFold(
 n_splits=config.cv, shuffle=True, random_state=config.seed)
 train_results, valid_results = [], []
best_models = []
for i_cv,(train_indices, valid_indices)in enumerate(
 splitter.split(train_dataset.df, train_dataset.df.target)) :
if config.cv_part is not None and i_cv >= config.cv_part:
break
train_tensor = train_dataset.build_labeled_dataset(train_indices)
 valid_tensor = train_dataset.build_labeled_dataset(valid_indices)
 valid_iter = DataLoader(
 valid_tensor, batch_size=config.batchsize_valid)
 model = models.pop(0)
 model = model.to_device(config.device)
 model_snapshots = []
optimizer = torch.optim.Adam(model.parameters() , config.lr)
 train_result = ClassificationResult('train', config.outdir, str(i_cv))
valid_result = ClassificationResult('valid', config.outdir, str(i_cv))
batchsize = config.batchsize
for epoch in range(config.epochs):
if epoch in config.scale_batchsize:
batchsize *= 2
print(f'Batchsize: {batchsize}')
 epoch_start = time.time()
sampler = None
train_iter = DataLoader(
 train_tensor, sampler=sampler, drop_last=True,
batch_size=batchsize, shuffle=sampler is None)
 _summary = []
for i, batch in enumerate(
 tqdm(train_iter, desc='train', leave=False)) :
model.train()
optimizer.zero_grad()
loss, output = model.calc_loss(*batch)
 loss.backward()
optimizer.step()
train_result.add_record(**output)
 train_result.calc_score(epoch)
 _summary.append(train_result.summary.iloc[-1])
 if epoch >= config.validate_from:
for i, batch in enumerate(
 tqdm(valid_iter, desc='valid', leave=False)) :
model.eval()
loss, output = model.calc_loss(*batch)
 valid_result.add_record(**output)
 valid_result.calc_score(epoch)
 _summary.append(valid_result.summary.iloc[-1])
 _model = deepcopy(model)
 _model.threshold = valid_result.summary.threshold[epoch]
model_snapshots.append(_model)
 summary = pd.DataFrame(_summary ).set_index('name')
 epoch_time = time.time() - epoch_start
pbar = '
tqdm.write(f'
{pbar} cv: {i_cv} / {config.cv}, epoch {epoch}, '
f'time: {epoch_time}')
 tqdm.write(str(summary))
train_results.append(train_result)
 valid_results.append(valid_result)
 best_indices = valid_result.summary.fbeta.argsort() [::-1]
best_models.extend([model_snapshots[i] for i in
best_indices[:config.ensembler_n_snapshots]] )<train_model>
A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = _a
a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = np.log(test.FVC.values[test.Patient == p])-(test.Weeks.values[test.Patient == p])* a
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
train_X, train_X2, train_t = \
train_dataset.X, train_dataset.X2, train_dataset.t
ensembler = Ensembler(config, best_models, valid_results)
 ensembler.fit(train_X, train_X2, train_t)
 scores = dict(
 valid_fbeta=np.array([r.best_fbeta for r in valid_results] ).mean() ,
valid_epoch=np.array([r.best_epoch for r in valid_results] ).mean() ,
threshold_cv=ensembler.threshold_cv,
threshold=ensembler.threshold,
elapsed_time=time.time() - start,
)<compute_train_metric>
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = np.exp(w*A_test[p] + B_test[p])
 sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] = P_test[p] - np.exp(A_test[p] * abs(WEEK[p] - w))
if config.holdout:
test_X, test_X2, test_t = \
test_dataset.X, test_dataset.X2, test_dataset._t
y, t = ensembler.predict_proba(test_X, test_X2), test_t
y_pred = y > ensembler.threshold
y_pred_cv = y > ensembler.threshold_cv
result = classification_metrics(y_pred, t)
 result_cv = classification_metrics(y_pred_cv, t)
 result_theoretical = classification_metrics(y, t)
 scores.update(dict(
 test_fbeta=result['fbeta'],
test_fbeta_cv=result_cv['fbeta'],
test_fbeta_theoretical=result_theoretical['fbeta'],
test_threshold_theoretical=result_theoretical['threshold'], 
))<save_to_csv>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False)
 img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
submit_y = ensembler.predict(submit_dataset.X, submit_dataset.X2)
 submit_df['prediction'] = submit_y
submit_df = submit_df[['qid', 'prediction']]
submit_df.to_csv(config.outdir / 'submission.csv', index=False )<concatenate>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
for i,(train, valid)in enumerate(zip(train_results, valid_results)) :
df = pd.concat([train.summary, valid.summary])
 df = df.set_index(['name', df.index], drop=True)
 display(df )<create_dataframe>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
pd.DataFrame([scores] )<import_modules>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
tqdm.pandas() 
<load_from_csv>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", train.shape)
 print("Test shape : ", test.shape )<feature_engineering>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))<prepare_x_and_y>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
embed_size = 300
max_features = None
maxlen = 72
X = train["question_text"].fillna("_na_" ).values
X_test = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features, filters='')
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 X_test = tokenizer.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 Y = train['target'].values
sub = test[['qid']]<set_options>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8375), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
del train, test
gc.collect()<statistical_test>
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
NFOLD = 5
kf = KFold(n_splits=NFOLD )
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean(( embedding_matrix_1, embedding_matrix_3), axis=0)
 del embedding_matrix_1, embedding_matrix_3
gc.collect()
np.shape(embedding_matrix )<statistical_test>
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )<choose_model_class>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
def capsule() :
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNLSTM(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=5, routings=2, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.1 )(x)
 x = BatchNormalization()(x)
 x = Capsule(num_capsule=10, dim_capsule=5, routings=2, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.1 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model<compute_test_metric>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2<split>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
kfold = StratifiedKFold(n_splits=5, random_state=1, shuffle=True)
 bestscore = []
y_test = np.zeros(( X_test.shape[0],))
for i,(train_index, valid_index)in enumerate(kfold.split(X, Y)) :
X_train, X_val, Y_train, Y_val = X[train_index], X[valid_index], Y[train_index], Y[valid_index]
filepath="weights_best.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
model = capsule()
if i == 0:print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=6, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks, 
)
 model.load_weights(filepath)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 y_test += np.squeeze(model.predict([X_test], batch_size=1024, verbose=2)) /5
f1, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))
bestscore.append(threshold )<save_to_csv>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>np.mean(bestscore)).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )<choose_model_class>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
 class AttentionWeightedAverage(Layer):

 def __init__(self, return_attention=False, **kwargs):
self.init = initializers.get('uniform')
 self.supports_masking = True
self.return_attention = return_attention
super(AttentionWeightedAverage, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(ndim=3)]
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[2], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self.trainable_weights = [self.W]
super(AttentionWeightedAverage, self ).build(input_shape)
 def call(self, x, mask=None):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1]))
ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))
if mask is not None:
mask = K.cast(mask, K.floatx())
 ai = ai * mask
att_weights = ai /(K.sum(ai, axis=1, keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, att_weights]
return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[2]
if self.return_attention:
return [(input_shape[0], output_len),(input_shape[0], input_shape[1])]
return(input_shape[0], output_len)
 def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
 else:
return None
spell_model = gensim.models.KeyedVectors.load_word2vec_format('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')
 words = spell_model.index2word
w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i,letter in enumerate(word)if i == 0 or letter != word[i-1]])
 def load_glove(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
def load_fasttext(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
def load_para(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
def build_model(embedding_matrix, nb_words, embedding_size=300):
inp = Input(shape=(max_length,))
x = Embedding(nb_words, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.3 )(x)
 x1 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x1)
 max_pool1 = GlobalMaxPooling1D()(x1)
 max_pool2 = GlobalMaxPooling1D()(x2)
 conc = Concatenate()([max_pool1, max_pool2])
 predictions = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=predictions)
 adam = optimizers.Adam(lr=learning_rate)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
 return model
start_time = time.time()
print("Loading data...")
 train = pd.read_csv('.. /input/train.csv' ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv' ).fillna(' ')
 train_text = train['question_text']
test_text = test['question_text']
text_list = pd.concat([train_text, test_text])
 y = train['target'].values
num_train_data = y.shape[0]
print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
print("Spacy NLP...")
 nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(text_list, n_threads = 2)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
gc.collect()
train_word_sequences = word_sequences[:num_train_data]
test_word_sequences = word_sequences[num_train_data:]
print("--- %s seconds ---" %(time.time() - start_time))
max_length = 55
embedding_size = 600
learning_rate = 0.001
batch_size = 512
num_epoch = 4
train_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')
 test_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')
 print(train_word_sequences[:1])
 print(test_word_sequences[:1])
 pred_prob = np.zeros(( len(test_word_sequences),), dtype=np.float32)
 start_time = time.time()
print("Loading embedding matrix...")
 embedding_matrix_glove, nb_words = load_glove(word_dict, lemma_dict)
 embedding_matrix_fasttext, nb_words = load_fasttext(word_dict, lemma_dict)
 embedding_matrix = np.concatenate(( embedding_matrix_glove, embedding_matrix_fasttext), axis=1)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time() 
<train_model>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
print("Start training...")
 model = build_model(embedding_matrix, nb_words, embedding_size)
 model.summary()<train_model>
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.25*df1['Confidence'] + 0.75*df2['Confidence']
df.head()
<import_modules><EOS> 
df.to_csv('submission.csv', index=False )