Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<choose_model_class>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, torch.optim.Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs<choose_model_class>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
class SimpleRNN(nn.Module):
def __init__(self, config: Config, embeddings):
super(SimpleRNN, self ).__init__()
self.c = config
self.embedding = embeddings
self.lstm = LSTMLayer(input_size=self.c.embedding_size,
hidden_size=self.c.hidden_size,
num_layers=self.c.num_layers,
dropout_rate=self.c.layer_dropout)
 self.gru = GRULayer(input_size=self.c.hidden_size*2,
hidden_size=self.c.hidden_size,
num_layers=self.c.num_gru_layers,
dropout_rate=self.c.layer_dropout)
 self.cell_dropout = nn.Dropout(self.c.layer_dropout)
 self.linear = nn.Linear(self.c.dense_size[0], self.c.dense_size[1])
 self.batch_norm = torch.nn.BatchNorm1d(self.c.dense_size[1])
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(self.c.layer_dropout)
 self.out = nn.Linear(self.c.dense_size[1], self.c.output_size)
 def forward(self, x, features):
h_embedding = self.embedding(x)
 o_lstm, h_lstm = self.lstm(h_embedding)
 o_gru, h_gru = self.gru(o_lstm)
 avg_pool = torch.mean(o_gru, 1)
 max_pool, _ = torch.max(o_gru, 1)
 h_lstm = self.cell_dropout(torch.cat(h_lstm.split(1, 0), -1 ).squeeze(0))
h_gru = self.cell_dropout(torch.cat(h_gru.split(1, 0), -1 ).squeeze(0))
concat = torch.cat([h_lstm, h_gru, avg_pool, max_pool, features], 1)
 concat = self.linear(concat)
 concat = self.batch_norm(concat)
 concat = self.relu(concat)
 concat = self.dropout(concat)
 out = self.out(concat)
 return out<split>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
def cut_length(data, mask):
max_length = data.shape[1]
transposed = torch.transpose(data, 1, 0)
 res =(transposed == mask ).all(1)
 for i, r in enumerate(res):
if r == 0:
break
data = data[:, -(max_length - i):]
return data<train_model>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
@timer
def training(train_x, train_f, train_y, test_x, test_f, c, embeddings):
splits = list(StratifiedKFold(n_splits=c.num_cv_splits, shuffle=True, random_state=SEED ).split(train_x, train_y))
x_test_cuda = torch.tensor(test_x, dtype=torch.long ).cuda(cuda_idx)
 f_test_cuda = torch.tensor(test_f, dtype=torch.float ).cuda(cuda_idx)
 test = torch.utils.data.TensorDataset(x_test_cuda, f_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=c.test_batch_size, shuffle=False)
 train_preds = np.zeros(( len(train_x)))
 test_preds = np.zeros(( len(test_x)))
 mask = torch.zeros(( c.max_length, 1), dtype=torch.long ).cuda(cuda_idx)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_x[train_idx], dtype=torch.long ).cuda(cuda_idx)
 f_train_fold = torch.tensor(train_f[train_idx], dtype=torch.float ).cuda(cuda_idx)
 y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda(cuda_idx)
 x_val_fold = torch.tensor(train_x[valid_idx], dtype=torch.long ).cuda(cuda_idx)
 f_val_fold = torch.tensor(train_f[valid_idx], dtype=torch.float ).cuda(cuda_idx)
 y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda(cuda_idx)
 embeddings.reset_weights()
model = SimpleRNN(c, embeddings)
 model.cuda(cuda_idx)
 model.embedding.original_embedding_weights.cuda(cuda_idx)
 loss_fn = torch.nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=c.learning_rate)
 scheduler = CyclicLR(optimizer, base_lr=c.learning_rate, max_lr=c.max_learning_rate,
step_size=c.clr_step_size, mode='exp_range', gamma=0.9999)
 train = torch.utils.data.TensorDataset(x_train_fold, f_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, f_val_fold, y_val_fold)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=c.test_batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(c.num_epochs):
train_loader = torch.utils.data.DataLoader(train, batch_size=c.batch_size[epoch], shuffle=True)
 start_time = time.time()
if epoch >= c.num_epochs - c.num_embedding_learn_epochs:
model.embedding.embeddings.weight.requires_grad = True
model.train()
avg_loss = 0.
 for x_batch, f_batch, y_batch in tqdm(train_loader, disable=True):
x_batch = cut_length(x_batch, mask)
 y_pred = model(x_batch, f_batch)
 scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , c.clip_grad)
 optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_x))
avg_val_loss = 0.
 for i,(x_batch, f_batch, y_batch)in enumerate(valid_loader):
x_batch = cut_length(x_batch, mask)
 y_pred = model(x_batch, f_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * c.test_batch_size:(i+1)* c.test_batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
for ps in optimizer.param_groups:
current_learning_rate = ps['lr']
elapsed_time = time.time() - start_time
print('Epoch {}/{} loss={:.4f} val_loss={:.4f} time={:.2f}s lr={:.6f}'.format(
 epoch + 1, c.num_epochs, avg_loss, avg_val_loss, elapsed_time, current_learning_rate))
for i,(x_batch, f_batch)in enumerate(test_loader):
x_batch = cut_length(x_batch, mask)
 y_pred = model(x_batch, f_batch ).detach()
test_preds_fold[i * c.test_batch_size:(i+1)* c.test_batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 return train_preds, test_preds<train_model>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
train_preds, test_preds = training(train_x, train_features, train_y, test_x, test_features, c, embeddings )<compute_test_metric>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
search_result = threshold_search(train_y, train_preds)
 search_result<save_to_csv>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
submission = test_df[['qid']].copy()
submission['prediction'] =(test_preds > search_result['threshold'] ).astype(int)
 submission.to_csv("submission.csv", index=False )<import_modules>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(0.5 )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/effnet-b5-30epochs-1/effnet_30.h5')
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
tqdm.pandas(desc='Progress')
<define_variables>
tr_p, vl_p = train_test_split(P,
shuffle=True,
train_size= 1 )
embed_size = 300
max_features = 120000
maxlen = 70
batch_size = 512
n_epochs = 5
n_splits = 5
SEED = 6017
debug =0<choose_model_class>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum' )<set_options>
subs = []
for model in models:
q = 0.5
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 1.1 and int(i[:-4])/ len(ldir)> -0.1:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
def seed_everything(seed=6017):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<features_selection>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
else:
embedding_vector = embeddings_index.get(word.capitalize())
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<define_variables>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<concatenate>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
def parallelize_apply(df,func,colname,num_process,newcolnames):
pool =Pool(processes=num_process)
 arraydata = pool.map(func,tqdm(df[colname].values))
pool.close()
newdf = pd.DataFrame(arraydata,columns = newcolnames)
 df = pd.concat([df,newdf],axis=1)
 return df
def parallelize_dataframe(df, func):
df_split = np.array_split(df, 4)
 pool = Pool(4)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df
def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df["lower_question_text"] = df["question_text"].apply(lambda x: x.lower())
 df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
def load_and_prec() :
if debug:
train_df = pd.read_csv(".. /input/train.csv")[:80000]
test_df = pd.read_csv(".. /input/test.csv")[:20000]
else:
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train = add_features(train_df)
 test = add_features(test_df)
 train = parallelize_dataframe(train_df, add_features)
 test = parallelize_dataframe(test_df, add_features)
 train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
features = train[['num_unique_words','words_vs_unique']].fillna(0)
 test_features = test[['num_unique_words','words_vs_unique']].fillna(0)
 ss = StandardScaler()
pc = PCA(n_components=5)
 ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
features = features[trn_idx]
return train_X, test_X, train_y, features, test_features, tokenizer.word_index<split>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
start = time.time()
x_train, x_test, y_train, features, test_features, word_index = load_and_prec()
print(time.time() -start )<normalization>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
seed_everything()
if debug:
paragram_embeddings = np.random.randn(120000,300)
 glove_embeddings = np.random.randn(120000,300)
 embedding_matrix = np.mean([glove_embeddings,paragram_embeddings], axis=0)
 else:
glove_embeddings = load_glove(word_index)
 paragram_embeddings = load_para(word_index)
 embedding_matrix = np.mean([glove_embeddings,paragram_embeddings], axis=0 )<choose_model_class>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs 
<categorify>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
class MyDataset(Dataset):
def __init__(self,dataset):
self.dataset = dataset
def __getitem__(self,index):
data,target = self.dataset[index]
return data,target,index
def __len__(self):
return len(self.dataset )<compute_train_metric>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
def pytorch_model_run_cv(x_train,y_train,features,x_test, model_obj, feats = False,clip = True):
seed_everything()
avg_losses_f = []
avg_val_losses_f = []
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(x_test)))
 splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED ).split(x_train, y_train))
for i,(train_idx, valid_idx)in enumerate(splits):
seed_everything(i*1000+i)
 x_train = np.array(x_train)
 y_train = np.array(y_train)
 if feats:
features = np.array(features)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
if feats:
kfold_X_features = features[train_idx.astype(int)]
kfold_X_valid_features = features[valid_idx.astype(int)]
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
model = copy.deepcopy(model_obj)
 model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 step_size = 300
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = MyDataset(torch.utils.data.TensorDataset(x_train_fold, y_train_fold))
valid = MyDataset(torch.utils.data.TensorDataset(x_val_fold, y_val_fold))
train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(train_loader):
if feats:
f = kfold_X_features[index]
y_pred = model([x_batch,f])
 else:
y_pred = model(x_batch)
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
if clip:
nn.utils.clip_grad_norm_(model.parameters() ,1)
 optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(x_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch,index)in enumerate(valid_loader):
if feats:
f = kfold_X_valid_features[index]
y_pred = model([x_batch,f] ).detach()
else:
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[index] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
if feats:
f = test_features[i * batch_size:(i+1)* batch_size]
y_pred = model([x_batch,f] ).detach()
else:
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
 return train_preds, test_preds<choose_model_class>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
class NeuralNet(nn.Module):
def __init__(self,hidden_size,lin_size, embedding_matrix=embedding_matrix):
super(NeuralNet, self ).__init__()
self.hidden_size = hidden_size
drp = 0.1
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 for name, param in self.lstm.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 for name, param in self.gru.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 self.linear = nn.Linear(hidden_size*6 + features.shape[1], lin_size)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(drp)
 self.out = nn.Linear(lin_size, 1)
 def forward(self, x):
h_embedding = self.embedding(x[0])
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, hh_gru = self.gru(h_lstm)
 hh_gru = hh_gru.view(-1, 2*self.hidden_size)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 f = torch.tensor(x[1], dtype=torch.float ).cuda()
conc = torch.cat(( hh_gru, avg_pool, max_pool,f), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out 
<normalization>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
seed_everything()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )<compute_train_metric>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 FE += ['age','week','BASE']
train_preds , test_preds = pytorch_model_run_cv(x_train,y_train,features,x_test,NeuralNet(90,16, embedding_matrix=embedding_matrix), feats = True )<compute_test_metric>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
search_result = threshold_search(y_train, train_preds)
 search_result<save_to_csv>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
if debug:
df_test = pd.read_csv(".. /input/test.csv")[:20000]
else:
df_test = pd.read_csv(".. /input/test.csv")
 submission = df_test[['qid']].copy()
submission['prediction'] = test_preds > search_result['threshold']
submission.to_csv('submission.csv', index=False )<choose_model_class>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
 class AttentionWeightedAverage(Layer):

 def __init__(self, return_attention=False, **kwargs):
self.init = initializers.get('uniform')
 self.supports_masking = True
self.return_attention = return_attention
super(AttentionWeightedAverage, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(ndim=3)]
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[2], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self.trainable_weights = [self.W]
super(AttentionWeightedAverage, self ).build(input_shape)
 def call(self, x, mask=None):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1]))
ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))
if mask is not None:
mask = K.cast(mask, K.floatx())
 ai = ai * mask
att_weights = ai /(K.sum(ai, axis=1, keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, att_weights]
return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[2]
if self.return_attention:
return [(input_shape[0], output_len),(input_shape[0], input_shape[1])]
return(input_shape[0], output_len)
 def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
 else:
return None
spell_model = gensim.models.KeyedVectors.load_word2vec_format('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')
 words = spell_model.index2word
w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i,letter in enumerate(word)if i == 0 or letter != word[i-1]])
 def load_glove(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
def load_fasttext(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
def load_para(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
def build_model(embedding_matrix, nb_words, embedding_size=300):
inp = Input(shape=(max_length,))
x = Embedding(nb_words, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.3 )(x)
 x1 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x1)
 max_pool1 = GlobalMaxPooling1D()(x1)
 max_pool2 = GlobalMaxPooling1D()(x2)
 conc = Concatenate()([max_pool1, max_pool2])
 predictions = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=predictions)
 adam = optimizers.Adam(lr=learning_rate)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
 return model
start_time = time.time()
print("Loading data...")
 train = pd.read_csv('.. /input/train.csv' ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv' ).fillna(' ')
 train_text = train['question_text']
test_text = test['question_text']
text_list = pd.concat([train_text, test_text])
 y = train['target'].values
num_train_data = y.shape[0]
print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
print("Spacy NLP...")
 nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(text_list, n_threads = 2)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
gc.collect()
train_word_sequences = word_sequences[:num_train_data]
test_word_sequences = word_sequences[num_train_data:]
print("--- %s seconds ---" %(time.time() - start_time))
max_length = 55
embedding_size = 600
learning_rate = 0.001
batch_size = 512
num_epoch = 4
train_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')
 test_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')
 print(train_word_sequences[:1])
 print(test_word_sequences[:1])
 pred_prob = np.zeros(( len(test_word_sequences),), dtype=np.float32)
 start_time = time.time()
print("Loading embedding matrix...")
 embedding_matrix_glove, nb_words = load_glove(word_dict, lemma_dict)
 embedding_matrix_fasttext, nb_words = load_fasttext(word_dict, lemma_dict)
 embedding_matrix = np.concatenate(( embedding_matrix_glove, embedding_matrix_fasttext), axis=1)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
print("Start training...")
 model = build_model(embedding_matrix, nb_words, embedding_size)
 model.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)
 pred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))
model.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)
 pred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))
del model, embedding_matrix_fasttext, embedding_matrix
gc.collect()
K.clear_session()
print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
print("Loading embedding matrix...")
 embedding_matrix_para, nb_words = load_para(word_dict, lemma_dict)
 embedding_matrix = np.concatenate(( embedding_matrix_glove, embedding_matrix_para), axis=1)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
print("Start training...")
 model = build_model(embedding_matrix, nb_words, embedding_size)
 model.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)
 pred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))
model.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)
 pred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))
print("--- %s seconds ---" %(time.time() - start_time))
submission = pd.DataFrame.from_dict({'qid': test['qid']})
 submission['prediction'] =(pred_prob>0.35 ).astype(int)
 submission.to_csv('submission.csv', index=False)
<load_from_csv>
NFOLD = 2
kf = KFold(n_splits=NFOLD )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv' )<data_type_conversions>
%%time
cnt = 0
EPOCHS = 600
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
def fixData(input_set):
input_set.rename(columns={'Country_Region':'Country'}, inplace=True)
 input_set.rename(columns={'Province_State':'State'}, inplace=True)
 input_set['Date'] = pd.to_datetime(input_set['Date'], infer_datetime_format=True)
 input_set['Date'] = input_set.Date.dt.strftime("%m%d")
 input_set['Date'] = input_set['Date'].astype(int)
 input_set["State"].fillna("",inplace=True)
 input_set["CountryState"] = input_set["Country"] + input_set["State"]
return input_set
train = fixData(train)
 test = fixData(test )<categorify>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
filterwarnings('ignore')
 label = LabelEncoder()
sub = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 CountryState = train.CountryState.unique()
for CS in CountryState:
train2 = train[train["CountryState"] == CS]
test2 = test[test["CountryState"] == CS]
train2.CountryState = label.fit_transform(train2.CountryState)
 X = train2[['CountryState', 'Date']]
Y = train2[['ConfirmedCases']]
eval_set = [(X,Y)]
model1 = boostmodel(learning_rate=0.3,silent=0, n_estimators=1000)
 model1.fit(X, Y,eval_set=eval_set,early_stopping_rounds=100)
 testX = test2[['CountryState', 'Date']]
testX.CountryState = label.fit_transform(testX.CountryState)
 ConfirmedCases_Pred = model1.predict(testX)
 X = train2[['CountryState', 'Date']]
Y = train2[['Fatalities']]
eval_set = [(X,Y)]
model2 = boostmodel(learning_rate=0.3,silent=0, n_estimators=1020)
 model2.fit(X, Y,eval_set=eval_set,early_stopping_rounds=100)
 testX = test2[['CountryState', 'Date']]
testX.CountryState = label.fit_transform(testX.CountryState)
 Fatalities_Pred = model2.predict(testX)
 XForecastId = test2.loc[:, 'ForecastId']
output = pd.DataFrame({'ForecastId': XForecastId, 'ConfirmedCases': ConfirmedCases_Pred, 'Fatalities': Fatalities_Pred})
 sub = pd.concat([sub, output], axis=0)
<prepare_output>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
sub.ConfirmedCases.apply(math.floor)
 sub.ForecastId = sub.ForecastId.astype('int')
 sub.ConfirmedCases = round(sub.ConfirmedCases,1)
 sub.Fatalities = round(sub.Fatalities,1)
 sub = sub[['ForecastId','ConfirmedCases','Fatalities']]<save_to_csv>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
sigma_mean = 60
if sigma_mean<sigma_mean:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
sub.to_csv("submission.csv",index=False )<load_from_csv>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
warnings.filterwarnings(action='ignore')
 def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'])
 feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 print(X_train)
 for day in sorted(feature_day,reverse = True):
feature_use = 'Number day from ' + str(day)+ ' case'
idx = X_train[X_train[feature_use] == 0].shape[0]
if(X_train[X_train[feature_use] > 0].shape[0] >= 20):
break
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()<save_to_csv>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False )<set_options>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
pd.plotting.register_matplotlib_converters()
%matplotlib inline
print('Setup complete' )<load_from_csv>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
covid_train=pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv', index_col='Id', parse_dates=['Date'])
 covid_test=pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv', index_col='ForecastId', parse_dates=['Date'])
 last_register= pd.to_datetime(covid_train["Date"].iloc[covid_train.shape[0]-1])
 print("Len train %d, Len test %d"%(covid_train.shape[0], covid_test.shape[0]))
print("Last train "Date": ", last_register )<feature_engineering>
df = df1[['Patient_Week']].copy()
df['FVC'] =(0.45*df1['FVC'] + 0.55*df2['FVC'])
 df['Confidence'] =(0.45*df1['Confidence'] + 0.55*df2['Confidence'])
 df.head()
<define_variables><EOS> 
df.to_csv('submission.csv', index=False )