Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<feature_engineering>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
%%time
token_name = {}
cnt = 1
for i in list(tokenizer.word_index.keys()):
if i in name:
token_name[i] = cnt
cnt += 1
else:
token_name[i] = 0
tokenizer.word_index = token_name<string_transform>
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error
from tensorflow_addons.optimizers import RectifiedAdam
from tensorflow.keras import Model
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.optimizers import Nadam
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from PIL import Image
X_train = train_df['transform'].values
X_val = val_df['transform'].values
y_train = train_df['target'].values
y_val = val_df['target'].values
X_test = test_df['transform'].values
X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train_name = sequence.pad_sequences(X_train, maxlen=maxlen, padding='post',truncating='post')
 x_val_name = sequence.pad_sequences(X_val, maxlen=maxlen, padding='post', truncating='post')
 x_test_name = sequence.pad_sequences(X_test, maxlen=maxlen, padding='post', truncating='post' )<feature_engineering>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )
train_df['token'] = list(x_train)
 train_df['token_name'] = list(x_train_name )<define_variables>
seed_everything(42 )
mask_zeros = np.ones(( name.shape[0] + 1, 600))
mask_zeros[0] = 0<import_modules>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
BatchNormalization, GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate, Conv1D, Multiply, Add<set_options>
Dropout_model = 0.3856
FVC_weight = 0.2
Confidence_weight = 0.2
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<categorify>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
class GlobalMinPooling1D(Layer):
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
return K.min(x, axis=1)
 def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[2])
<categorify>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
class GlobalSumPooling1D(Layer):
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
return K.sum(x, axis=1)
 def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[2])
<choose_model_class>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
def build_model(input_layer, input_layer_name, embedding_matrix):
x1 = Embedding(embedding_matrix.shape[0], 600, weights=[embedding_matrix], trainable= False )(input_layer)
 x2 = Embedding(name.shape[0] + 1, 600, trainable= True )(input_layer_name)
 x3 = Embedding(name.shape[0] + 1, 600, weights=[mask_zeros], trainable= False )(input_layer_name)
 x = Multiply()([x2, x3])
 x = Add()([x1, x])
 x = Bidirectional(CuDNNLSTM(128, kernel_initializer=he_uniform(seed=0), return_sequences=True))(x)
 x = SpatialDropout1D(0.2 )(x)
 y = Bidirectional(CuDNNGRU(128,kernel_initializer=he_uniform(seed=0), return_sequences=True))(x)
 a = GlobalAveragePooling1D()(y)
 b = GlobalMaxPooling1D()(y)
 c = GlobalMinPooling1D()(y)
 x = concatenate([a, b])
 x = Dense(32, activation="relu",kernel_initializer=he_uniform(seed=0))(x)
 x = Dense(1, activation="sigmoid",kernel_initializer=he_uniform(seed=0))(x)
 return x<compute_test_metric>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<split>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
train_df.reset_index(drop = True, inplace = True)
 val_df.reset_index(drop= True, inplace = True )<define_variables>
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.9 )
positive = train_df[train_df['target'] == 1][['token','token_name','target']]
negative = train_df[train_df['target'] == 0][['token','token_name','target']]<split>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 69)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
for C in range(4):
if(C == 0):
embedding_matrix = np.concatenate([embedding_matrix1, embedding_matrix2], axis=1)
 elif(C == 1):
embedding_matrix = np.concatenate([embedding_matrix1, embedding_matrix2], axis=1)
 elif(C == 2):
embedding_matrix = np.concatenate([embedding_matrix2, embedding_matrix1], axis=1)
 else:
embedding_matrix = np.concatenate([ embedding_matrix2, embedding_matrix1], axis=1)
 neg1, neg2 = train_test_split(negative, test_size = 0.5, random_state = C*100)
 df1, df2 = pd.concat([neg1,positive], ignore_index=True), pd.concat([neg2,positive], ignore_index=True)
 input_layer = Input(( 50,), name="i1")
 input_layer_name = Input(( 50,), name = "i2")
 output_layer = build_model(input_layer, input_layer_name, embedding_matrix)
 model = Model([input_layer, input_layer_name], output_layer)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 model.fit({"i1": np.stack(df1['token'].values), "i2": np.stack(df1['token_name'].values)},
df1['target'], batch_size=128, verbose=2,shuffle=True,\
epochs=1, validation_data=({"i1": x_val, "i2": x_val_name}, val_df['target']))
model.fit({"i1":x_train, "i2": x_train_name}, train_df['target'], batch_size=256, verbose=2,shuffle=True,\
epochs=1, validation_data=({"i1": x_val, "i2": x_val_name}, val_df['target']))
model.layers[2].trainable = False
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 model.fit({"i1":x_train, "i2": x_train_name}, train_df['target'], batch_size=512, verbose=2,shuffle=True,\
epochs=1, validation_data=({"i1": x_val, "i2": x_val_name}, val_df['target']))
test_df[C] = model.predict({"i1": x_test, "i2": x_test_name}, batch_size=1024 ).flatten() 
<feature_engineering>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
test_df['preds'] =(test_df[0] + test_df[1] + test_df[2] + test_df[3])/4<save_to_csv>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
y_te =(test_df['preds'] > 0.35 ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )<import_modules>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
import re
import time
import gc
import random
import os
import math
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, precision_recall_fscore_support
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import torch
import torch.nn as nn
import torch.utils.data
import torch.nn.functional as F<set_options>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
def seed_torch(seed=1130):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = False
SEED = 1130
seed_torch(SEED )<define_variables>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
embed_size = 300
max_features = 196534
maxlen = 72
log_interval = 250
batch_size = 512<concatenate>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
num_partitions = 20
num_cores = psutil.cpu_count()
print('number of cores:', num_cores)
 def df_parallelize_run(df, func):
df_split = np.array_split(df, num_partitions)
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df<define_variables>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
puncts = ['!', '?', '.', ',', '-', "'", '"', ':', ')', '(', '|', ';', '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '÷', 'π',
'है', 'है',
]
puncts_str = ''.join(puncts)[5:]
puncts = ['!', '?', '.', ',', '-']
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_quotation(x):
x = str(x)
 if "'s" in x:
x = x.replace("'s", "")
 if "s'" in x:
x = x.replace("s'", "")
 if "n't" in x:
x = x.replace("n't", " not")
 if "'ve" in x:
x = x.replace("'ve", " have")
 return x<define_variables>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
mispell_dict = {
"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have",
"couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not",
"hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did",
"how'd'y": "how do you", "how'll": "how will", "how's": "how is", "i'd": "i would", "i'd've": "i would have",
"i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would",
"it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us",
"ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have",
"must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not",
"needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have",
"shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would",
"she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is",
"should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have",
"so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is",
"there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have",
"they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would",
"we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have",
"weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are",
"what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did",
"where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is",
"who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have",
"y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are",
"y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will",
"you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center',
'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater',
'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',
'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist',
'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many',
'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation',
'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum',
'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota',
'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp',
'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization',
'quoran': 'quora', 'quorans': 'quora', 'brexit': 'british exit', 'quoras':'quora', 'fortnite': 'video game',
'pubg': 'video game', 'redmi': 'cell phone', 'jinping': 'chinese president', 'lyft': 'uber', 'θ': 'theta', 'σ': 'sigma'
}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<feature_engineering>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
def preprocess(text):

 text = text.lower()
text = clean_text(text)
 text = replace_typical_misspell(text)
 text = clean_quotation(text)
 return text
def text_clean_wrapper(df):
df["question_text"] = df["question_text"].apply(preprocess)
 return df<load_from_csv>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df = df_parallelize_run(train_df, text_clean_wrapper)
 test_df = df_parallelize_run(test_df, text_clean_wrapper)
 train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features, filters=puncts_str)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index<randomize_order>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
emb_mean, emb_std = -0.005838499, 0.48782197
embedding_matrix = np.random.normal(emb_mean, emb_std,(len(word_index), embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8")as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= len(word_index):
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
emb_mean, emb_std = -0.0053247833, 0.49346462
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8", errors='ignore')as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= max_features:
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
emb_mean, emb_std = -0.0033469985, 0.109855495
embedding_matrix = np.random.normal(emb_mean, emb_std,(len(word_index), embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8")as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= len(word_index):
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
tqdm.pandas()
start_time = time.time()
train_X, test_X, train_y, word_index = load_and_prec()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_para(word_index)
 embedding_matrix_3 = load_fasttext(word_index)
 total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
print(np.shape(embedding_matrix_1))<categorify>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
class RNNModel(nn.Module):
def __init__(self, rnn_type, input_size, hidden_size):
super(RNNModel, self ).__init__()
self.rnn = getattr(nn, rnn_type )(input_size, hidden_size, bidirectional=True, batch_first=True)
 def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.rnn(x)
 class LSTM_TextCNN(nn.Module):
def __init__(self, hidden_size, embedding_matrix, embed_size, embedding_dropout=0.5, initialization=True):
super(LSTM_TextCNN, self ).__init__()
self.embedding = nn.Embedding(len(word_index), embed_size, padding_idx=0)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(embedding_dropout)
 self.lstm = RNNModel("LSTM", embed_size, hidden_size)
 self.lstm.init_weights()
self.conv1 = nn.Conv1d(hidden_size*2, hidden_size, 1, bias=False)
 self.bn1 = nn.BatchNorm1d(hidden_size)
 self.conv2 = nn.Conv1d(hidden_size*2, hidden_size, 2, bias=False)
 self.bn2 = nn.BatchNorm1d(hidden_size)
 self.conv3 = nn.Conv1d(hidden_size*2, hidden_size, 3, bias=False)
 self.bn3 = nn.BatchNorm1d(hidden_size)
 self.conv4 = nn.Conv1d(hidden_size*2, hidden_size, 4, bias=False)
 self.bn4 = nn.BatchNorm1d(hidden_size)
 self.relu = nn.ReLU(inplace=True)
 self.merge = nn.Linear(hidden_size*4, hidden_size)
 self.dropout = nn.Dropout(0.1)
 self.bn_merge = nn.BatchNorm1d(hidden_size)
 self.out = nn.Linear(hidden_size, 1)
 if initialization:
self.init_weights()
def init_weights(self):
self.out.bias.data.fill_(0)
 nn.init.kaiming_normal_(self.out.weight)
 def forward(self, x):
embed = self.embedding(x)
 out = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(embed, 0)))
 out, _ = self.lstm(embed)
 out = out.transpose(1, 2)
 out1 = self.bn1(self.relu(self.conv1(out)))
 out1 = F.adaptive_max_pool1d(out1, 1)
 out2 = self.bn2(self.relu(self.conv2(out)))
 out2 = F.adaptive_max_pool1d(out2, 1)
 out3 = self.bn3(self.relu(self.conv3(out)))
 out3 = F.adaptive_max_pool1d(out3, 1)
 out4 = self.bn4(self.relu(self.conv4(out)))
 out4 = F.adaptive_max_pool1d(out4, 1)
 conc = torch.squeeze(torch.cat(( out1, out2, out3, out4), 1))
conc = self.relu(self.merge(conc))
conc = self.bn_merge(self.dropout(conc))
out = self.out(conc)
 return out
class LSTM_GRU(nn.Module):
def __init__(self, hidden_size, embedding_matrix, embed_size, embedding_dropout=0.1):
super(LSTM_GRU, self ).__init__()
self.hidden_size = hidden_size
self.embedding_matrix = embedding_matrix
self.embedding = nn.Embedding(len(word_index), embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(embedding_dropout)
 self.lstm = RNNModel("LSTM", embed_size, hidden_size)
 self.lstm.init_weights()
self.gru = RNNModel("GRU", hidden_size*2, hidden_size)
 self.gru.init_weights()
self.linear = nn.Linear(4*hidden_size, 16)
 self.relu = nn.ReLU(inplace=True)
 self.dropout = nn.Dropout(0.0)
 self.bn = nn.BatchNorm1d(16)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.bn(self.dropout(conc))
out = self.out(conc)
 return out<split>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED ).split(train_X, train_y))<compute_test_metric>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
def sigmoid(x):
return 1 /(1 + np.exp(-x))<compute_train_metric>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
def threshold_search_submit(y_proba, ratio):
best_threshold = 0
best_score = np.Inf
for threshold in tqdm([i * 0.001 for i in range(1000)]):
score = np.abs(( y_proba > threshold ).mean() - ratio)
 if score < best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'dist': best_score, 'ratio': ratio}
return search_result
def threshold_search(y_proba, y_true):
best_threshold = 0
best_score = 0
scores = []
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 scores.append(score)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
plt.figure(figsize=(12,9))
plt.plot([i * 0.01 for i in range(100)], scores)
 plt.plot(best_threshold, best_score, "xr", label="Best threshold")
 plt.show()
return search_result<choose_model_class>
%%time
cnt = 0
EPOCHS = 855
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
class CosineLRWithRestarts() :

 def __init__(self, optimizer, batch_size, epoch_size, restart_period=100,
t_mult=2, last_epoch=-1, eta_threshold=1000, verbose=False):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if last_epoch == -1:
for group in optimizer.param_groups:
group.setdefault('initial_lr', group['lr'])
 else:
for i, group in enumerate(optimizer.param_groups):
if 'initial_lr' not in group:
raise KeyError("param 'initial_lr' is not specified "
"in param_groups[{}] when resuming an"
" optimizer".format(i))
self.base_lrs = list(map(lambda group: group['initial_lr'],
optimizer.param_groups))
self.last_epoch = last_epoch
self.batch_size = batch_size
self.epoch_size = epoch_size
self.eta_threshold = eta_threshold
self.t_mult = t_mult
self.verbose = verbose
self.base_weight_decays = list(map(lambda group: group['weight_decay'],
optimizer.param_groups))
self.restart_period = restart_period
self.restarts = 0
self.t_epoch = -1
def _schedule_eta(self):

 eta_min = 0
eta_max = 1
if self.restarts <= self.eta_threshold:
return eta_min, eta_max
else:
d = self.restarts - self.eta_threshold
k = d * 0.09
return(eta_min + k, eta_max - k)
 def get_lr(self, t_cur):
eta_min, eta_max = self._schedule_eta()
eta_t =(eta_min + 0.5 *(eta_max - eta_min)
 *(1.+ math.cos(math.pi * 
(t_cur / self.restart_period))))
weight_decay_norm_multi = math.sqrt(self.batch_size / 
(self.epoch_size *
self.restart_period))
lrs = [base_lr * eta_t for base_lr in self.base_lrs]
weight_decays = [base_weight_decay * eta_t * weight_decay_norm_multi
for base_weight_decay in self.base_weight_decays]
if self.t_epoch % self.restart_period < self.t_epoch:
if self.verbose:
print("Restart at epoch {}".format(self.last_epoch))
self.restart_period *= self.t_mult
self.restarts += 1
self.t_epoch = 0
return zip(lrs, weight_decays)
 def _set_batch_size(self):
d, r = divmod(self.epoch_size, self.batch_size)
 batches_in_epoch = d + 2 if r > 0 else d + 1
self.batch_increment = iter(torch.linspace(0, 1, batches_in_epoch))
def step(self):
self.last_epoch += 1
self.t_epoch += 1
self._set_batch_size()
self.batch_step()
def batch_step(self):
t_cur = self.t_epoch + next(self.batch_increment)
 for param_group,(lr, weight_decay)in zip(self.optimizer.param_groups,
self.get_lr(t_cur)) :
param_group['lr'] = lr
param_group['weight_decay'] = weight_decay<compute_train_metric>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
def f1_loss(logits, labels):
__small_value = 1e-6
beta = 1
batch_size = logits.size() [0]
p = torch.sigmoid(logits)
 l = labels
num_pos = torch.sum(p, 1)+ __small_value
num_pos_hat = torch.sum(l, 1)+ __small_value
tp = torch.sum(l * p, 1)
 precise = tp / num_pos
recall = tp / num_pos_hat
fs =(1 + beta * beta)* precise * recall /(beta * beta * precise + recall + __small_value)
 loss = fs.sum() / batch_size
return(1 - loss)
 class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, logit, target):
max_val =(-logit ).clamp(min=0)
 loss = logit - logit * target + max_val +(( -max_val ).exp() +(-logit - max_val ).exp() ).log()
invprobs = F.logsigmoid(-logit *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
if len(loss.size())==2:
loss = loss.sum(dim=1)
 return loss.mean()
def lovasz_grad(gt_sorted):

 p = len(gt_sorted)
 gts = gt_sorted.sum()
intersection = gts - gt_sorted.float().cumsum(0)
 union = gts +(1 - gt_sorted ).float().cumsum(0)
 jaccard = 1.- intersection / union
if p > 1:
jaccard[1:p] = jaccard[1:p] - jaccard[0:-1]
return jaccard
def lovasz_hinge_flat(logits, labels):

 logits = logits.squeeze()
labels = labels.squeeze()
if len(labels)== 0:
return logits.sum() * 0.
 signs = 2.* labels.float() - 1.
 errors =(1.- logits * Variable(signs))
errors_sorted, perm = torch.sort(errors, dim=0, descending=True)
 perm = perm.data
gt_sorted = labels[perm]
grad = lovasz_grad(gt_sorted)
 loss = torch.dot(F.elu(errors_sorted)+ 1, Variable(grad))
return loss<create_dataframe>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
train_epochs = 4
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )<prepare_x_and_y>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model1 = LSTM_TextCNN(64, np.concatenate(( embedding_matrix_1, embedding_matrix_3), axis=1), embed_size*2, embedding_dropout=0.5)
 model2 = LSTM_GRU(64, np.mean([embedding_matrix_1, embedding_matrix_2], axis=0), embed_size, embedding_dropout=0.1)
 model1.cuda()
model2.cuda()
loss_fn1 = torch.nn.BCEWithLogitsLoss()
loss_fn2 = f1_loss
optimizer1 = torch.optim.Adam(model1.parameters() , lr=0.0035)
 scheduler1 = CosineLRWithRestarts(optimizer1, batch_size, len(x_train_fold), restart_period=4, t_mult=1, verbose=True)
 optimizer2 = torch.optim.Adam(model2.parameters() , lr=0.0035)
 scheduler2 = CosineLRWithRestarts(optimizer2, batch_size, len(x_train_fold), restart_period=4, t_mult=1, verbose=True)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(train_epochs):
start_time = time.time()
scheduler1.step()
scheduler2.step()
model1.train()
model2.train()
for batch_idx,(x_batch, y_batch)in enumerate(train_loader):
y_pred1 = model1(x_batch)
 y_pred2 = model2(x_batch)
 loss1 = loss_fn1(y_pred1, y_batch)+ loss_fn2(y_pred1, y_batch)
 loss2 = loss_fn1(y_pred2, y_batch)+ loss_fn2(y_pred2, y_batch)
 optimizer1.zero_grad()
optimizer2.zero_grad()
loss1.backward()
loss2.backward()
optimizer1.step()
optimizer2.step()
scheduler1.batch_step()
scheduler2.batch_step()
model1.eval()
model2.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred1 = model1(x_batch ).detach()
y_pred2 = model2(x_batch ).detach()
valid_preds_fold[i * batch_size:(i+1)* batch_size] =(
 sigmoid(y_pred1.cpu().numpy())[:, 0] +
sigmoid(y_pred2.cpu().numpy())[:, 0] 
)/ 2
search_result = threshold_search(valid_preds_fold, train_y[valid_idx])
 valid_yhat_fold =(valid_preds_fold > search_result['threshold'] ).astype(int)
 valid_precision, valid_recall, valid_fscore, _ = precision_recall_fscore_support(train_y[valid_idx], valid_yhat_fold, average='binary')
 elapsed_time = time.time() - start_time
print('Epoch {}/{} | val_precision={:.5f} | val_recall={:.5f} | val_f1={:.5f} | threshold={:.2f} | time={:.2f}s'.format(
 epoch, train_epochs - 1, valid_precision, valid_recall, valid_fscore, search_result['threshold'], elapsed_time))
for i,(x_batch,)in enumerate(test_loader):
y_pred1 = model1(x_batch ).detach()
y_pred2 = model2(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] =(
 sigmoid(y_pred1.cpu().numpy())[:, 0] +
sigmoid(y_pred2.cpu().numpy())[:, 0] 
)/ 2
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<compute_train_metric>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
search_result = threshold_search(train_preds, train_y)
 train_yhat =(train_preds > search_result['threshold'] ).astype(int)
 train_precision, train_recall, train_fscore, _ = precision_recall_fscore_support(train_y, train_yhat, average='binary')
 print('oof_precision={:.5f} | oof_recall={:.5f} | oof_f1={:.5f} | threshold={:.2f}'.format(train_precision, train_recall, train_fscore, search_result['threshold']))<compute_train_metric>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
ratio =(train_yhat.mean() +(test_preds > search_result['threshold'] ).mean())/ 2
search_result = threshold_search_submit(test_preds, ratio)
 print(search_result )<save_to_csv>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
tqdm.pandas(desc='Progress')
<define_variables>
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()
embed_size = 300
max_features = 120000
maxlen = 70
batch_size = 512
n_epochs = 5
n_splits = 5
SEED = 1029
debug =0<choose_model_class>
df.to_csv('submission.csv', index=False )
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum' )<set_options>
df_issa1 = df.copy()
def seed_everything(seed=10):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<features_selection>
Dropout_model = 0.3856
FVC_weight = 0.2
Confidence_weight = 0
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
else:
embedding_vector = embeddings_index.get(word.capitalize())
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<feature_engineering>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42)
 config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config)
 train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding")
<define_variables>
def expandFVC_Percent(DS):
DS['FVC_Percent'] = DS['FVC'] / DS['Percent']
return DS
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
if bool(re.search(r'\d', x)) :
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
<concatenate>
def expandFE(data_set):
data_set['MinWeeks'] = data_set.groupby('Patient')['Weeks'].transform('min')
 sub_tr = data_set.loc[data_set['Weeks']==data_set['MinWeeks']]
sub_tr['MinFVC'] = sub_tr['FVC']
sub_tr = sub_tr[['Patient', 'MinFVC']]
merged = data_set.merge(sub_tr, on='Patient', how='outer')
 data_set=merged
data_set['MinFVC'] =(data_set['MinFVC'] - data_set['MinFVC'].min())/(data_set['MinFVC'].max() -data_set['MinFVC'].min())
 print(data_set.sample(5))
return data_set
def parallelize_apply(df,func,colname,num_process,newcolnames):
pool =Pool(processes=num_process)
 arraydata = pool.map(func,tqdm(df[colname].values))
pool.close()
newdf = pd.DataFrame(arraydata,columns = newcolnames)
 df = pd.concat([df,newdf],axis=1)
 return df
def parallelize_dataframe(df, func):
df_split = np.array_split(df, 4)
 pool = Pool(4)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df
def count_regexp_occ(regexp="", text=None):

 return len(re.findall(regexp, text))
def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df["lower_question_text"] = df["question_text"].apply(lambda x: x.lower())
 df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
def load_and_prec() :
if debug:
train_df = pd.read_csv(".. /input/train.csv")[:80000]
test_df = pd.read_csv(".. /input/test.csv")[:20000]
else:
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)

train = parallelize_dataframe(train_df, add_features)
 test = parallelize_dataframe(test_df, add_features)
 train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
features = train[['num_unique_words','words_vs_unique']].fillna(0)
 test_features = test[['num_unique_words','words_vs_unique']].fillna(0)
 ss = StandardScaler()
pc = PCA(n_components=5)
 ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
features = features[trn_idx]
return train_X, test_X, train_y, features, test_features, tokenizer.word_index 
<categorify>
train = expandFE(train)
start = time.time()
x_train, x_test, y_train, features, test_features, word_index = load_and_prec()
print(time.time() -start )<normalization>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=8):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return math.ceil(len(self.keys)/ float(self.batch_size))
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
self.train_data[k].sort()
mx_len = len(self.train_data[k])
 i = np.random.choice(self.train_data[k][mx_len//3:mx_len//3*2], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x, a, tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
seed_everything()
if debug:
paragram_embeddings = np.random.randn(120000,300)
 glove_embeddings = np.random.randn(120000,300)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)
 else:
glove_embeddings = load_glove(word_index)
 paragram_embeddings = load_para(word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0 )<choose_model_class>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(5,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 return model
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs 
<categorify>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 subs = []
q = -1
class MyDataset(Dataset):
def __init__(self,dataset):
self.dataset = dataset
def __getitem__(self,index):
data,target = self.dataset[index]
return data,target,index
def __len__(self):
return len(self.dataset )<find_best_model_class>
q = -1
print(q )
def pytorch_model_run_cv(x_train,y_train,features,x_test, model_obj, feats = False,clip = True):
seed_everything()
avg_losses_f = []
avg_val_losses_f = []
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(x_test)))
 global_test_saver = []
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED ).split(x_train, y_train))
for i,(train_idx, valid_idx)in enumerate(splits):
seed_everything(i*1000+i)
 x_train = np.array(x_train)
 y_train = np.array(y_train)
 if feats:
features = np.array(features)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
if feats:
kfold_X_features = features[train_idx.astype(int)]
kfold_X_valid_features = features[valid_idx.astype(int)]
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
model = copy.deepcopy(model_obj)
 model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 step_size = 300
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = MyDataset(torch.utils.data.TensorDataset(x_train_fold, y_train_fold))
valid = MyDataset(torch.utils.data.TensorDataset(x_val_fold, y_val_fold))
train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(train_loader):
if feats:
f = kfold_X_features[index]
y_pred = model([x_batch,f])
 else:
y_pred = model(x_batch)
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
if clip:
nn.utils.clip_grad_norm_(model.parameters() ,1)
 optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(x_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch,index)in enumerate(valid_loader):
if feats:
f = kfold_X_valid_features[index]
y_pred = model([x_batch,f] ).detach()
else:
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[index] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
if feats:
f = test_features[i * batch_size:(i+1)* batch_size]
y_pred = model([x_batch,f] ).detach()
else:
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 global_test_saver.append(test_preds_fold)
 diff = np.zeros([n_splits,n_splits])
 for ii in range(n_splits):
for jj in range(ii,n_splits):
diff[ii,jj] = int(np.sum(np.abs(global_test_saver[ii] - global_test_saver[jj])))
 diff_sum = np.sum(diff)
 print(diff)
 print(diff_sum /(n_splits)/(n_splits - 1)* 2 /len(global_test_saver[0]))
for ii in range(n_splits):
for jj in range(ii,n_splits):
diff[ii,jj] = int(np.sum(np.power(global_test_saver[ii] - global_test_saver[jj],2)))
 diff_sum = np.sum(diff)
 print(diff)
 print(diff_sum /(n_splits)/(n_splits - 1)* 2 /len(global_test_saver[0]))
print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
 return train_preds, test_preds<choose_model_class>
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
class Alex_NeuralNet_Meta(nn.Module):
def __init__(self,hidden_size,lin_size, embedding_matrix=embedding_matrix):
super(Alex_NeuralNet_Meta, self ).__init__()
self.hidden_size = hidden_size
drp = 0.1
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 for name, param in self.lstm.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 for name, param in self.gru.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 self.linear = nn.Linear(hidden_size*6 + features.shape[1], lin_size)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(drp)
 self.out = nn.Linear(lin_size, 1)
 self.bn = nn.BatchNorm1d(16, momentum=0.5)
 def forward(self, x):
h_embedding = self.embedding(x[0])
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, hh_gru = self.gru(h_lstm)
 hh_gru = hh_gru.view(-1, 2*self.hidden_size)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 f = torch.tensor(x[1], dtype=torch.float ).cuda()
conc = torch.cat(( hh_gru, avg_pool, max_pool,f), 1)
 conc = self.relu(self.linear(conc))
conc = self.bn(conc)
 conc = self.dropout(conc)
 out = self.out(conc)
 return out 
<normalization>
N = len(subs)
 print(N)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N)
 sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False)
 img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
print(img_sub.sample(5))
def sigmoid(x):
return 1 /(1 + np.exp(-x))
seed_everything()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )<compute_train_metric>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
train_preds , test_preds = pytorch_model_run_cv(x_train,y_train,features,x_test,Alex_NeuralNet_Meta(70,16, embedding_matrix=embedding_matrix), feats = True )<compute_test_metric>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 data['FVC_Percent'] = data['FVC'] / data['Percent']
FE += ['age','percent','week','BASE']
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
print(FE )
def bestThresshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)) :
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))
return delta , tmp[2]
delta, _ = bestThresshold(y_train,train_preds )<save_to_csv>
y = tr['FVC'].values
z = tr[FE].values
print(z.shape)
 ze = sub[FE].values
print(ze.shape)
 nh = z.shape[1]
print(nh)
 pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
net = make_model(nh)
 print(net.summary())
 print(net.count_params())
 NFOLD = 5
kf = KFold(n_splits=NFOLD )
if debug:
df_test = pd.read_csv(".. /input/test.csv")[:20000]
else:
df_test = pd.read_csv(".. /input/test.csv")
 submission = df_test[['qid']].copy()
submission['prediction'] =(test_preds > delta ).astype(int)
 submission.to_csv('submission.csv', index=False )<import_modules>
cnt = 0
EPOCHS = 800
y = y.astype(np.float32)
 for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
tqdm.pandas(desc='Progress')
<define_variables>
df_issa2 = df.copy()
embed_size = 300
max_features = 120000
maxlen = 70
batch_size = 512
n_epochs = 5
n_splits = 5
SEED = 1029<set_options>
df_issa1.to_csv('df_issa1.csv', index=False)
 df_issa2.to_csv('df_issa2.csv', index=False )
def seed_everything(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<features_selection>
x = 0.6
df = df_issa1.copy()
df['FVC'] = x*df_issa1['FVC'] +(1-x)*df_issa2['FVC']
df['Confidence'] = x*df_issa1['Confidence'] +(1-x)*df_issa2['Confidence']
df.head()
<load_from_csv><EOS> 
df.to_csv('submission.csv', index=False )