Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<choose_model_class>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
<compute_test_metric>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
<import_modules>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
import re
import time
import gc
import random
import os
import math
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, precision_recall_fscore_support
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import torch
import torch.nn as nn
import torch.utils.data
import torch.nn.functional as F<set_options>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
def seed_torch(seed=21330):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = False
SEED = 21330
seed_torch(SEED )<define_variables>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
embed_size = 300
max_features = 196534
maxlen = 72
log_interval = 250
batch_size = 512<concatenate>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
num_partitions = 20
num_cores = psutil.cpu_count()
print('number of cores:', num_cores)
 def df_parallelize_run(df, func):
df_split = np.array_split(df, num_partitions)
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df<define_variables>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
puncts = ['!', '?', '.', ',', '-', "'", '"', ':', ')', '(', '|', ';', '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '÷', 'π',
'है', 'है',
]
puncts_str = ''.join(puncts)[5:]
puncts = ['!', '?', '.', ',', '-']
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_quotation(x):
x = str(x)
 if "'s" in x:
x = x.replace("'s", "")
 if "s'" in x:
x = x.replace("s'", "")
 if "n't" in x:
x = x.replace("n't", " not")
 if "'ve" in x:
x = x.replace("'ve", " have")
 return x<define_variables>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(0.5 )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
mispell_dict = {
"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have",
"couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not",
"hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did",
"how'd'y": "how do you", "how'll": "how will", "how's": "how is", "i'd": "i would", "i'd've": "i would have",
"i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would",
"it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us",
"ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have",
"must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not",
"needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have",
"shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would",
"she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is",
"should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have",
"so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is",
"there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have",
"they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would",
"we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have",
"weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are",
"what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did",
"where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is",
"who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have",
"y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are",
"y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will",
"you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center',
'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater',
'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',
'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist',
'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many',
'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation',
'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum',
'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota',
'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp',
'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization',
'quoran': 'quora', 'quorans': 'quora', 'brexit': 'british exit', 'quoras':'quora', 'fortnite': 'video game',
'pubg': 'video game', 'redmi': 'cell phone', 'jinping': 'chinese president', 'lyft': 'uber', 'θ': 'theta', 'σ': 'sigma'
}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<feature_engineering>
tr_p, vl_p = train_test_split(P,
shuffle=True,
train_size= 0.8 )
def preprocess(text):

 text = text.lower()
text = clean_text(text)
 text = replace_typical_misspell(text)
 text = clean_quotation(text)
 return text
def text_clean_wrapper(df):
df["question_text"] = df["question_text"].apply(preprocess)
 return df<load_from_csv>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df = df_parallelize_run(train_df, text_clean_wrapper)
 test_df = df_parallelize_run(test_df, text_clean_wrapper)
 train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features, filters=puncts_str)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index<randomize_order>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
emb_mean, emb_std = -0.005838499, 0.48782197
embedding_matrix = np.random.normal(emb_mean, emb_std,(len(word_index), embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8")as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= len(word_index):
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
emb_mean, emb_std = -0.0053247833, 0.49346462
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8", errors='ignore')as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= max_features:
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
emb_mean, emb_std = -0.0033469985, 0.109855495
embedding_matrix = np.random.normal(emb_mean, emb_std,(len(word_index), embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8")as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= len(word_index):
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
tqdm.pandas()
start_time = time.time()
train_X, test_X, train_y, word_index = load_and_prec()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_para(word_index)
 embedding_matrix_3 = load_fasttext(word_index)
 total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
print(np.shape(embedding_matrix_1))<categorify>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
class RNNModel(nn.Module):
def __init__(self, rnn_type, input_size, hidden_size):
super(RNNModel, self ).__init__()
self.rnn = getattr(nn, rnn_type )(input_size, hidden_size, bidirectional=True, batch_first=True)
 def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.rnn(x)
 class LSTM_TextCNN(nn.Module):
def __init__(self, hidden_size, embedding_matrix, embed_size, embedding_dropout=0.5, initialization=True):
super(LSTM_TextCNN, self ).__init__()
self.embedding = nn.Embedding(len(word_index), embed_size, padding_idx=0)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(embedding_dropout)
 self.lstm = RNNModel("LSTM", embed_size, hidden_size)
 self.lstm.init_weights()
self.conv1 = nn.Conv1d(hidden_size*2, hidden_size, 1, bias=False)
 self.bn1 = nn.BatchNorm1d(hidden_size)
 self.conv2 = nn.Conv1d(hidden_size*2, hidden_size, 2, bias=False)
 self.bn2 = nn.BatchNorm1d(hidden_size)
 self.conv3 = nn.Conv1d(hidden_size*2, hidden_size, 3, bias=False)
 self.bn3 = nn.BatchNorm1d(hidden_size)
 self.conv4 = nn.Conv1d(hidden_size*2, hidden_size, 4, bias=False)
 self.bn4 = nn.BatchNorm1d(hidden_size)
 self.relu = nn.ReLU(inplace=True)
 self.merge = nn.Linear(hidden_size*4, hidden_size)
 self.dropout = nn.Dropout(0.1)
 self.bn_merge = nn.BatchNorm1d(hidden_size)
 self.out = nn.Linear(hidden_size, 1)
 if initialization:
self.init_weights()
def init_weights(self):
self.out.bias.data.fill_(0)
 nn.init.kaiming_normal_(self.out.weight)
 def forward(self, x):
embed = self.embedding(x)
 out = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(embed, 0)))
 out, _ = self.lstm(embed)
 out = out.transpose(1, 2)
 out1 = self.bn1(self.relu(self.conv1(out)))
 out1 = F.adaptive_max_pool1d(out1, 1)
 out2 = self.bn2(self.relu(self.conv2(out)))
 out2 = F.adaptive_max_pool1d(out2, 1)
 out3 = self.bn3(self.relu(self.conv3(out)))
 out3 = F.adaptive_max_pool1d(out3, 1)
 out4 = self.bn4(self.relu(self.conv4(out)))
 out4 = F.adaptive_max_pool1d(out4, 1)
 conc = torch.squeeze(torch.cat(( out1, out2, out3, out4), 1))
conc = self.relu(self.merge(conc))
conc = self.bn_merge(self.dropout(conc))
out = self.out(conc)
 return out
class LSTM_GRU(nn.Module):
def __init__(self, hidden_size, embedding_matrix, embed_size, embedding_dropout=0.1):
super(LSTM_GRU, self ).__init__()
self.hidden_size = hidden_size
self.embedding_matrix = embedding_matrix
self.embedding = nn.Embedding(len(word_index), embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(embedding_dropout)
 self.lstm = RNNModel("LSTM", embed_size, hidden_size)
 self.lstm.init_weights()
self.gru = RNNModel("GRU", hidden_size*2, hidden_size)
 self.gru.init_weights()
self.linear = nn.Linear(4*hidden_size, 16)
 self.relu = nn.ReLU(inplace=True)
 self.dropout = nn.Dropout(0.0)
 self.bn = nn.BatchNorm1d(16)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.bn(self.dropout(conc))
out = self.out(conc)
 return out<split>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED ).split(train_X, train_y))<compute_test_metric>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
def sigmoid(x):
return 1 /(1 + np.exp(-x))<compute_train_metric>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
train_ratio = 80810 / 1306122
public_test_ratio = 3376 / 56370
submit_ratio =(80810 + 3376)/(1306122 + 56370)
 def threshold_search_oof(y_proba, y_true=None, mode="train"):
if mode == "train":
ratio = train_ratio
elif mode == "test":
ratio = public_test_ratio
elif mode == "submit":
ratio = submit_ratio
else:
raise ValueError("Invalid mode")
 best_threshold = 0
best_score = np.Inf
for threshold in tqdm([i * 0.001 for i in range(1000)]):
score = np.abs(( y_proba > threshold ).mean() - ratio)
 if score < best_score:
best_threshold = threshold
best_score = score
if mode == "train":
f1 = f1_score(y_true=y_true, y_pred=y_proba > best_threshold)
 search_result = {'threshold': best_threshold, 'dist': best_score, 'f1': f1}
return search_result
else:
search_result = {'threshold': best_threshold, 'dist': best_score}
return search_result
def threshold_search(y_proba, y_true):
best_threshold = 0
best_score = 0
scores = []
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 scores.append(score)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
plt.figure(figsize=(12,9))
plt.plot([i * 0.01 for i in range(100)], scores)
 plt.plot(best_threshold, best_score, "xr", label="Best threshold")
 plt.show()
return search_result<choose_model_class>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
class CosineLRWithRestarts() :

 def __init__(self, optimizer, batch_size, epoch_size, restart_period=100,
t_mult=2, last_epoch=-1, eta_threshold=1000, verbose=False):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if last_epoch == -1:
for group in optimizer.param_groups:
group.setdefault('initial_lr', group['lr'])
 else:
for i, group in enumerate(optimizer.param_groups):
if 'initial_lr' not in group:
raise KeyError("param 'initial_lr' is not specified "
"in param_groups[{}] when resuming an"
" optimizer".format(i))
self.base_lrs = list(map(lambda group: group['initial_lr'],
optimizer.param_groups))
self.last_epoch = last_epoch
self.batch_size = batch_size
self.epoch_size = epoch_size
self.eta_threshold = eta_threshold
self.t_mult = t_mult
self.verbose = verbose
self.base_weight_decays = list(map(lambda group: group['weight_decay'],
optimizer.param_groups))
self.restart_period = restart_period
self.restarts = 0
self.t_epoch = -1
def _schedule_eta(self):

 eta_min = 0
eta_max = 1
if self.restarts <= self.eta_threshold:
return eta_min, eta_max
else:
d = self.restarts - self.eta_threshold
k = d * 0.09
return(eta_min + k, eta_max - k)
 def get_lr(self, t_cur):
eta_min, eta_max = self._schedule_eta()
eta_t =(eta_min + 0.5 *(eta_max - eta_min)
 *(1.+ math.cos(math.pi * 
(t_cur / self.restart_period))))
weight_decay_norm_multi = math.sqrt(self.batch_size / 
(self.epoch_size *
self.restart_period))
lrs = [base_lr * eta_t for base_lr in self.base_lrs]
weight_decays = [base_weight_decay * eta_t * weight_decay_norm_multi
for base_weight_decay in self.base_weight_decays]
if self.t_epoch % self.restart_period < self.t_epoch:
if self.verbose:
print("Restart at epoch {}".format(self.last_epoch))
self.restart_period *= self.t_mult
self.restarts += 1
self.t_epoch = 0
return zip(lrs, weight_decays)
 def _set_batch_size(self):
d, r = divmod(self.epoch_size, self.batch_size)
 batches_in_epoch = d + 2 if r > 0 else d + 1
self.batch_increment = iter(torch.linspace(0, 1, batches_in_epoch))
def step(self):
self.last_epoch += 1
self.t_epoch += 1
self._set_batch_size()
self.batch_step()
def batch_step(self):
t_cur = self.t_epoch + next(self.batch_increment)
 for param_group,(lr, weight_decay)in zip(self.optimizer.param_groups,
self.get_lr(t_cur)) :
param_group['lr'] = lr
param_group['weight_decay'] = weight_decay<compute_train_metric>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
def f1_loss(logits, labels):
__small_value = 1e-6
beta = 1
batch_size = logits.size() [0]
p = torch.sigmoid(logits)
 l = labels
num_pos = torch.sum(p, 1)+ __small_value
num_pos_hat = torch.sum(l, 1)+ __small_value
tp = torch.sum(l * p, 1)
 precise = tp / num_pos
recall = tp / num_pos_hat
fs =(1 + beta * beta)* precise * recall /(beta * beta * precise + recall + __small_value)
 loss = fs.sum() / batch_size
return(1 - loss)
 class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, logit, target):
max_val =(-logit ).clamp(min=0)
 loss = logit - logit * target + max_val +(( -max_val ).exp() +(-logit - max_val ).exp() ).log()
invprobs = F.logsigmoid(-logit *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
if len(loss.size())==2:
loss = loss.sum(dim=1)
 return loss.mean()
def lovasz_grad(gt_sorted):

 p = len(gt_sorted)
 gts = gt_sorted.sum()
intersection = gts - gt_sorted.float().cumsum(0)
 union = gts +(1 - gt_sorted ).float().cumsum(0)
 jaccard = 1.- intersection / union
if p > 1:
jaccard[1:p] = jaccard[1:p] - jaccard[0:-1]
return jaccard
def lovasz_hinge_flat(logits, labels):

 logits = logits.squeeze()
labels = labels.squeeze()
if len(labels)== 0:
return logits.sum() * 0.
 signs = 2.* labels.float() - 1.
 errors =(1.- logits * Variable(signs))
errors_sorted, perm = torch.sort(errors, dim=0, descending=True)
 perm = perm.data
gt_sorted = labels[perm]
grad = lovasz_grad(gt_sorted)
 loss = torch.dot(F.elu(errors_sorted)+ 1, Variable(grad))
return loss<create_dataframe>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
train_epochs = 4
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )<prepare_x_and_y>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model1 = LSTM_TextCNN(64, np.concatenate(( embedding_matrix_1, embedding_matrix_3), axis=1), embed_size*2, embedding_dropout=0.5)
 model2 = LSTM_GRU(64, np.mean([embedding_matrix_1, embedding_matrix_2], axis=0), embed_size, embedding_dropout=0.1)
 model1.cuda()
model2.cuda()
loss_fn1 = torch.nn.BCEWithLogitsLoss()
loss_fn2 = f1_loss
optimizer1 = torch.optim.Adam(model1.parameters() , lr=0.0035)
 scheduler1 = CosineLRWithRestarts(optimizer1, batch_size, len(x_train_fold), restart_period=4, t_mult=1, verbose=True)
 optimizer2 = torch.optim.Adam(model2.parameters() , lr=0.0035)
 scheduler2 = CosineLRWithRestarts(optimizer2, batch_size, len(x_train_fold), restart_period=4, t_mult=1, verbose=True)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(train_epochs):
start_time = time.time()
scheduler1.step()
scheduler2.step()
model1.train()
model2.train()
for batch_idx,(x_batch, y_batch)in enumerate(train_loader):
y_pred1 = model1(x_batch)
 y_pred2 = model2(x_batch)
 loss1 = loss_fn1(y_pred1, y_batch)+ loss_fn2(y_pred1, y_batch)
 loss2 = loss_fn1(y_pred2, y_batch)+ loss_fn2(y_pred2, y_batch)
 optimizer1.zero_grad()
optimizer2.zero_grad()
loss1.backward()
loss2.backward()
optimizer1.step()
optimizer2.step()
scheduler1.batch_step()
scheduler2.batch_step()
model1.eval()
model2.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred1 = model1(x_batch ).detach()
y_pred2 = model2(x_batch ).detach()
valid_preds_fold[i * batch_size:(i+1)* batch_size] =(
 sigmoid(y_pred1.cpu().numpy())[:, 0] +
sigmoid(y_pred2.cpu().numpy())[:, 0] 
)/ 2
search_result = threshold_search(valid_preds_fold, train_y[valid_idx])
 valid_yhat_fold =(valid_preds_fold > search_result['threshold'] ).astype(int)
 valid_precision, valid_recall, valid_fscore, _ = precision_recall_fscore_support(train_y[valid_idx], valid_yhat_fold, average='binary')
 elapsed_time = time.time() - start_time
print('Epoch {}/{} | val_precision={:.5f} | val_recall={:.5f} | val_f1={:.5f} | threshold={:.2f} | time={:.2f}s'.format(
 epoch, train_epochs - 1, valid_precision, valid_recall, valid_fscore, search_result['threshold'], elapsed_time))
for i,(x_batch,)in enumerate(test_loader):
y_pred1 = model1(x_batch ).detach()
y_pred2 = model2(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] =(
 sigmoid(y_pred1.cpu().numpy())[:, 0] +
sigmoid(y_pred2.cpu().numpy())[:, 0] 
)/ 2
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<compute_train_metric>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
search_result = threshold_search(train_preds, train_y)
 train_yhat =(train_preds > search_result['threshold'] ).astype(int)
 train_precision, train_recall, train_fscore, _ = precision_recall_fscore_support(train_y, train_yhat, average='binary')
 print('oof_precision={:.5f} | oof_recall={:.5f} | oof_f1={:.5f} | threshold={:.2f}'.format(train_precision, train_recall, train_fscore, search_result['threshold']))<save_to_csv>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False )<feature_engineering>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
LOCAL = False
seed =1018
if LOCAL:
FILE_DIR = './'
else:
FILE_DIR = '.. /input'
@contextmanager
def timer(task_name="timer"):
print("----{} started".format(task_name))
t0 = time.time()
yield
print("----{} done in {:.0f} seconds".format(task_name, time.time() - t0))
train = pd.read_csv(f'{FILE_DIR}/train.csv')
 test = pd.read_csv(f'{FILE_DIR}/test.csv')
 def do_rnns(dfs):
if dfs[2]:
split = -1
train_df, test_df = dfs[0], dfs[1]
print(train_df.shape, test_df.shape)
 test_idx = list(test_df.qid.values)
 test_df.drop('qid', axis=1, inplace=True)
 training_labels = train_df['target'].values
if split>0:
testing_labels = test_df['target'].values
def get_special_feats(df):
df['question_text'] = df['question_text'].apply(lambda x:str(x))
df['total_length'] = df['question_text'].apply(len)
 df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
features = df[['caps_vs_length', 'words_vs_unique']].fillna(0 ).values
gc.collect()
return features
print('START SPECIAL FEATS')
 train_features = get_special_feats(train_df)
 test_features = get_special_feats(test_df)
 print('END SPECIAL FEATS')
 ss = StandardScaler()
ss.fit(np.vstack(( train_features, test_features)))
 train_features = ss.transform(train_features)
 test_features = ss.transform(test_features)
 print('SCALE SPECIAL FEATS')
 punctuations = [
',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'*', '+', '\', '•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™',
'›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢',
'²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼',
'⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅',
'‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬',
'❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '\xa0', '高', '端', '大', '气', '上', '档', '次', '_', '½', 'π', '
'小', '鹿', '乱', '撞', '成', '语', 'ë', 'à', 'ç', '@', 'ü', 'č', 'ć', 'ž', 'đ', '°', 'द', 'े', 'श', '्', 'र', 'ो', 'ह', 'ि', 'प', 'स', 'थ', 'त', 'न', 'व', 'ा', 'ल', 'ं', '林', '彪', '€', '\u200b', '˚', 'ö', '~', '—', '越', '人', 'च', 'म', 'क',
'ु', 'य', 'ी', 'ê', 'ă', 'ễ', '∞', '抗', '日', '神', '剧', '，', '\uf02d', '–', 'ご', 'め', 'な', 'さ', 'い', 'す', 'み', 'ま', 'せ', 'ん', 'ó', 'è', '£', '¡', 'ś', '≤', '¿', 'λ', '魔', '法', '师', '）', 'ğ', 'ñ', 'ř', '그', '자', '식', '멀',
'쩡', '다', '인', '공', '호', '흡', '데', '혀', '밀', '어', '넣', '는', '거', '보', '니', 'ǒ', 'ú', '️', 'ش', 'ه', 'ا', 'د', 'ة', 'ل', 'ت', 'َ', 'ع', 'م', 'ّ', 'ق', 'ِ', 'ف', 'ي', 'ب', 'ح', 'ْ', 'ث', '³', '饭', '可', '以', '吃', '话', '不', '讲',
'∈', 'ℝ', '爾', '汝', '文', '言', '∀', '禮', 'इ', 'ब', 'छ', 'ड', '़', 'ʒ', '有', '「', '寧', '錯', '殺', '一', '千', '絕', '放', '過', '」', '之', '勢', '㏒', '㏑', 'ू', 'â', 'ω', 'ą', 'ō', '精', '杯', 'í', '生', '懸', '命', 'ਨ', 'ਾ', 'ਮ', 'ੁ',
'₁', '₂', 'ϵ', 'ä', 'к', 'ɾ', '\ufeff', 'ã', '©', '\x9d', 'ū', '™', '＝', 'ù', 'ɪ', 'ŋ', 'خ', 'ر', 'س', 'ن', 'ḵ', 'ā']
def clean_text(text):
text = str(text)
 for p in punctuations:
if p in text:
text = text.replace(p, f' {p} ')
 return(text)
 print('CLEAN TEXT')
 train_df['question_text'] = train_df['question_text'].apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].apply(lambda x: clean_text(x))
print('FINISH CLEANING TEXT')
 contractions_dict = {
"ain't": "am not",
"aren't": "are not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he had",
"he'd've": "he would have",
"he'll": "he shall",
"he'll've": "he shall have",
"he's": "he has",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how has",
"I'd": "I had",
"I'd've": "I would have",
"I'll": "I will",
"I'll've": "I will have",
"I'm": "I am",
"I've": "I have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so is",
"that'd": "that had",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": " you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have",
'隶书': 'Lishu', 'तपस्': 'Tapse', 'ให้': 'give', '宗教官': 'Religious officer', '没有神一样的对手': 'No god-like opponent', 'יהודיות': 'Jewish women', '我明明是中国人': 'I am obviously Chinese.', 'お早う': 'Good morning', 'मेरे': 'my', 'পরীক্ষা': 'Test', 'נעכטן': 'Yesterday', '中华民国': 'Republic of China', 'तकनीकी': 'Technical', 'यहाँ': 'here', '奇兵隊': 'Qibing', 'かﾟ': 'Or', '知识分子': 'Intellectuals', 'ごめなさい': "I'm sorry", '하기를': 'To', 'गाढवाचे': 'Donkey', 'इबादत': 'Worship', '千金': 'Thousand gold', 'ельзи': 'elzhi', '低端人口': 'Low-end population', '맛저': 'Mazur', 'दानि': 'Danias', 'юродство': 'foolishness', 'τον': 'him', '素质': 'Quality', '王晓菲': 'Wang Xiaofei', 'माहेर': 'Maher', 'لمصارى': 'For my advice', '送客！': 'see a visitor out!', '然而': 'however', 'जिग्यासा': 'Jigyaasa', '都市': 'city', 'ஜோடி': 'Pair', 'لاني': 'See', 'пельмени': 'dumplings', '请不要误会': "Please don't misunderstand", 'люис': 'Lewis', '不送！': 'Do not send!', 'के': 'Of', 'मस्का': 'Mascara', 'вoyѕ': 'voyce', 'बुन्ना': 'Bunna', '战略支援部队': 'Strategic support force', '平埔族': 'Pingpu', 'فهمت؟': 'I see?', 'कयामत': 'Apocalypse', 'ức': 'memory', 'ᗰeᑎ': 'ᗰe ᑎ', 'सेकना': 'Sexting', 'धकेलना': 'Shove', 'পারি': 'We can', 'مقام': 'Official', 'बेसति': 'Baseless', '歪果仁研究协会': 'Hazelnut Research Association', '短信': 'SMS', 'всегда': 'is always', '修身': 'Slim fit', 'إنَّ': 'that', '不是民国': 'Not the Republic of China', '写的好': 'Written well', 'õhtupoolik': 'afternoon', 'तालीन': 'Training', 'और': 'And', 'щит': 'shield', 'ᗩtteᑎtiᐯe': 'ᗩtte ᑎ ti ᐯ e', 'اسکوپیه': 'Script', 'çomar': 'fido', '和製英語': 'Japanglish', '吊着命': 'Hanging', 'много': 'much', 'समुराय': 'Samurai', 'भटकना': 'Wander', 'مقاومة': 'resistance', '싱관없어': 'I have no one.', '修身養性': 'Self-cultivation', 'मटका': 'pot', 'θιοψʼ': 'thiós', 'ㅎㅎㅎㅎ': 'Hehe', 'تساوى': 'Equal', 'बाट': 'From', '不地道啊。': 'Not authentic.', 'контакт': 'contact', '이런것도': 'This', 'तै': 'The', 'मेल': 'similarity', 'álvarez': 'Alvarez', 'नुक्स': 'Damage', '口訣': 'Mouth', 'масло': 'butter', 'परम्परा': 'Tradition', '学会': 'learn', 'کردن': 'Make up', 'öd': 'bile', 'टशन': 'Tashan', 'つらやましす': 'I am cheating', 'чего': 'what', '为什么总有人能在shoplex里免费获取iphone等重金商品？': 'Why do people always get free access to iphone and other heavy commodities in shoplex?', 'श्री': 'Mr', 'प्रेषक': 'Sender', 'خواندن': 'Read', 'बदलेंगे': 'Will change', 'बीएड': 'Breed', 'अदा': 'Paid', 'फैलाना': 'spread', 'ബുദ്ധിമാനായ': 'Intelligent', '谢谢六佬': 'Thank you, Liu Wei', 'উপস্থাপন': 'Present', 'بكلها': 'All of them', 'जाए': 'Go', '无可挑剔': 'Impeccable', 'आना': 'come', '太阴': 'lunar', 'القط': 'The cat', '있네': 'It is.', 'कर्म': 'Karma', 'आड़': 'Shroud', 'įit': 'IIT', 'जशने': 'Jashnay', 'से': 'From', 'åkesson': 'Åkesson', '乳名': 'Milk name', '我看起来像韩国人吗': 'Do I look like a Korean?', 'тнan': 'tn', 'العظام': 'Bones', '暹罗': 'Siam', '小肥羊鍋店': 'Little fat sheep pot shop', 'করেন': 'Do it', 'हरामी': 'bastard', 'डाले': 'Cast', 'استراحة': 'Break', '射道': 'Shooting', 'επιστήμη': 'science', 'χράομαι': "I'm glad", '脚踏车': 'bicycle', 'हिलना': 'To move', 'đa': 'multi', '標點符號': 'Punctuation', 'मैं': 'I', '不能化生水谷精微': "Can't transform the water valley", 'निवाला': 'Boar', 'दर्शनाभिलाषी': 'Spectacular', 'বাছাই': 'Picking', 'どくぜんてき': 'Dokedari', 'पीवें।': 'Pieve.', 'ʻoumuamua': 'Priority', 'में': 'In', 'चलाऊ': 'Run', 'निपटाना': 'To settle', 'ごはん': 'rice', 'चार्ज': 'Charge', 'शिथिल': 'Loosely', 'ястребиная': 'yastrebina', 'ложись': 'lie down', '李银河': 'Li Yinhe', 'へも': 'Also', 'हुलिया': 'Hulia', 'ऊब': 'Bored', 'छाछ': 'Buttermilk', 'عن': 'About', 'नहीं': 'No', 'जीव': 'creatures', 'जेहाद': 'Jehad', 'νερό': 'water', '열여섯': 'sixteen', 'मार': 'Kill', '早就': 'Long ago', '《齐天大圣》for': '"Qi Tian Da Sheng" for', 'الجنس': 'Sex', 'مع': 'With', 'रंगरलियाँ': 'Color palettes', 'जोल': 'Jol', 'बुद्धी': 'Intelligence', '罗思成': 'Luo Sicheng', '独善的': 'Self-righteousness', 'धर्मः': 'Religion', '中国大陆的同胞们': 'Chinese compatriots', '愛してない': 'I do not love you', '日本語が上手ですね': 'Japanese is good', 'ओठ': 'Lips', '如果你希望表達你的觀點': 'If you want to express your opinion', 'देशवाशी': 'Countryman', 'καί': 'and', '阮铭': 'Yu Ming', '跑步跑': 'Running', 'हो': 'Ho', '「褒められる」': '"Praised"', 'నా': 'My', '\x10new': '?new', 'ゆいがどくそん': 'A graduate student', '白语': 'White language', 'वह': 'She', '白い巨塔': 'White big tower', '로리이고': 'Lori.', 'परि': 'Circumcision', 'æj': 'Aw', 'انضيع': 'Lost', '平民苗字必称義務令': 'Civilian Miaozi must be called an obligation', 'ਸਿੰਘ': 'Singh', 'уже': 'already', 'đầu': 'head', '雨热同期': 'Rain and heat', 'घुलना': 'Dissolve', 'мис': 'Miss', 'て下さいませんか': 'Could it be?', '猫头鹰': 'owl', 'चढ़': 'Climbing', '漢音': 'Hanyin', 'यही': 'This only', '只有猪一样的队友': 'Only pig-like teammates', 'übersoldaten': 'about soldiers', 'αθ': 'Ath', 'ουδεις': 'no', '党国合一': 'Party and country', 'रेड': 'Red', 'ढोलना': 'Drift', 'शाक': 'Shake', '같이': 'together', '攻克': 'capture', 'özalan': 'exclusive', 'काम': 'work', 'चाहना': 'Wish', '坚持一个中国原则': 'Adhere to the one-China principle', '배우고': 'Learn', '柴棍': 'Firewood stick', 'उसे': 'To him', 'на': 'on', '無手': 'No hands', 'čvrsnica': 'Čvrsnica', 'ज़ार': 'Tsar', 'ˆo': 'O', '後宮甄嬛傳': 'Harem of the harem', '意音文字': 'Sound character', 'बहारा': 'Bahara', 'てみる': 'Try', '老铁': 'Old iron', '野比のび太': 'Nobita Nobita', 'याद': 'remember', 'پیشی': 'Surpass', 'توقعك': 'Your expectation', 'はたち': 'Slender', 'فزت': 'I won', '伊藤': 'Ito', 'कलेजे': 'Liver', 'αγεν': 'ruin', 'ìmprovement': 'Improvement', 'では': 'Then.', 'பார்ப்பது': 'Viewing', '只留清气满乾坤': 'Only stay clear and full of energy', '이정도쯤': 'About this', 'すてん': 'Sponge', 'ما': 'What', '海南人の日本': "Hainan people's Japan", '小鹿乱撞': 'very excited', 'ῥωμαῖοι': 'ῥomaşii', 'वारी': 'Vary', 'भोज': 'Feast', '陈庚': 'Chen Geng', 'कट्टरवादि': 'Fanatic', '凌霸': 'Lingba', 'पार': 'The cross', 'कुचलना': 'To crush', 'रहा': 'Stayed', 'हम': 'We', 'бойся': 'be afraid', '大刀': 'Large sword', 'ন্ন': 'Done', '汽车': 'car', 'কে': 'Who', 'الکعبه': 'Alkaebe', '网络安全法': 'Cybersecurity law', 'नेपाली': 'Nepali', 'পদের': 'Position', '\x92t': '??t', 'रास': 'Ras', 'لكل': 'for every', 'शूद्राणां': 'Shudran', '此问题只是针对外国友人的一次统计': 'This question is only a statistic for foreign friends.', 'ឃើញឯកសារអំពីប្រវត្តិស្ត្រនៃប្រាសាទអង្គរវត្ត': 'See the history of Angkor Wat', 'спасибо': 'thank', 'رب': 'God', 'वजूद': 'Non existent', 'पकड़': 'Hold', 'बरहा': 'Baraha', '雾草': 'Fog grass', 'мощный': 'powerful', 'বৃদ্ধাশ্রম': 'Old age', 'आई': 'Mother', 'खड़ी': 'Steep', '\x1aaùõ\x8d': '? aùõ ??', 'āto': 'standard', '在冰箱裡': 'in the refrigerator', 'đời': 'life', 'लड़का': 'The boy', 'τὸ': 'the', 'مدرسان': 'Instructors', 'பறை': 'Drum', 'índia': 'India', 'दिया': 'Gave', '小篆': 'Xiao Yan', '唐樂': 'Tang Le', '米国': 'USA', '文言文': 'Classical Chinese', 'उवाच': 'Uwach', 'هدف': 'Target', 'घेरा': 'Cordon', 'झूम': 'Zoom', 'εσσα': 'all the same', '和製漢語': 'And making Chinese', 'ऐलोपैथिक': 'Allopathic', 'создала': 'has created', 'إدمان': 'addiction', '游泳游': 'Swimming tour', '狮子头': 'Lion head', 'दिये': 'Given', 'पास': 'near', 'εντjα': 'in', 'まする': 'To worship', 'हाल': 'condition', '짱깨': 'Chin', 'জল': 'Water', 'चीज': 'thing', 'îmwe': 'one', '胡江南': 'Hu Jiangnan', 'तमाम': 'All', 'कट्टर': 'Hardcore', '魔法师': 'Magician', 'బుూ\u200c': 'Buu', 'चनचल': 'Chanchal', 'सीधा': 'Straightforward', '不要人夸颜色好': "Don't exaggerate the color", 'हिमालय': 'Himalaya', 'सिंह': 'Lion', 'خراميشو': 'Wastewater', 'мoѕт': 'the moment', 'কোথায়': 'Where?', 'نصف': 'Half', 'رائع': 'Wonderful', 'добрый': 'kind', 'ज़र्रा': 'Zarra', 'يعقوب': 'Yacoub', 'सम्झौता': 'Agreement', 'ān': 'yes', '했다': 'did', 'ἀριστοκράτης': 'aristocrat', 'çan': 'Bell', '太阳': 'sun', 'सर': 'head', 'गुरु': 'Master', '嘉定': 'Jiading', '故乡': 'home', '安倍晋三': 'Shinzo Abe', 'मच्छर': 'Mosquito', 'सभी': 'All', '成語': 'idiom', '唯我独尊': 'Only me', 'นะค่ะ': 'Yes', 'นั่นคุณกำลังทำอะไร': 'What are you doing', 'ın': 's', 'الشامبيونزليج': 'Shamballons', '抗日神剧': 'Anti-Japanese drama', '小妹': 'Little sister', 'çok': 'very', 'लोड': 'Load', 'साहित्यिक': 'Literary', 'लाल।': 'Red.', 'فى': 'in a', '絕不放過一人」之勢': 'Never let go of one person', '国家主席': 'National president', '异议': 'objection', 'अनुस्वार': 'Anuswasar', 'តើបងប្អូនមានមធ្យបាយអ្វីខ្លះដើម្បីរក': 'What are some ways to find out', '允≱ၑご搢': '≱ ≱ ≱ 搢', '黒髪': 'Black hair', '自戳双目': 'Self-poke binocular', 'βροχέως': 'rainy', 'ని': 'The', '一带一路': 'Belt and Road', 'śliwińska': 'Śliwińska', 'κατὰ': 'against', '打古人名': 'Hit the ancient name', 'குறை': 'Low', 'üjin': 'Uji', 'öppning': 'opening', 'अन्जल': 'Anjal', '이와': 'And', 'आविष्कार': 'Invention', 'غنج': 'Gnostic', 'рассвет': 'dawn', 'होना': 'Happen', 'বাড়ি': 'Home', '入定': 'Enter', 'भेड़': 'The sheep', 'देगा': 'Will give', 'ब्रह्मा': 'Brahma', 'बीन': 'Bean', 'χράω': "I'm glad", 'तीन': 'three', 'ضرار': 'Drar', 'विराजमान': 'Seated', 'सैलाब': 'Salab', 'συνηθείας': 'communication', '구경하고': 'To see', 'चुल्ल': 'Chool', '红宝书': 'Red book', '羡慕和嫉妒是不一样的。': 'Envy and jealousy are not the same.', 'मेरा': 'my', 'कलटि': 'Katiti', 'हिमाती': 'Himalayan', 'ಸಾಧು': 'Sadhu', 'عطيتها': 'Her gift', 'छान': 'Nice', 'เกาหลี': 'Korea', 'íntruments': 'instruments', 'يتحمل': 'Bear', 'रुस्तम': 'Rustam', 'बरात': 'Baraat', 'रंग': 'colour', '나이': 'age', 'פרפר': 'butterfly', '老乡': 'Hometown', '謝謝': 'Thank you', 'í‰lbƒ': '‰ in lbƒ', 'हरजाई': 'Harjai', 'পেতে': 'Get to', '「勉強': '"study', 'रड़कना': 'To cry', '清真诊所': 'Halal clinic', 'أفضل': 'Best', 'استطيع': 'I can', 'नाकतोडा': 'Nag', 'बड़ि': 'Elder', 'वैद्य': 'Vaidya', 'أنَّ': 'that', 'いたロリィis': 'There was Lory', 'ìn': 'print', '本人堅持一個中國的原則': 'I adhere to the one-China principle', 'океан': 'ocean', 'बाद': 'after', '禮儀': 'etiquette', 'सिषेविरे': 'Sisavir', 'अमृत': 'Honeydew', 'بدو': 'run', 'ὅς': 'that is', 'छोटा': 'small', 'स्वभावप्रभवैर्गुणै': 'Nature is bad', 'ענג': 'Tight', 'שלמה': 'Complete', 'डोरे': 'Dore', '我害怕得毛骨悚然': 'I am afraid of creeps.', 'झोलि': 'Jolly', 'единадесет': 'eleven', 'என்ன': 'What', 'आयुरवेद': 'Ayurveda', '堵天下悠悠之口': 'Blocking the mouth of the world', 'الجمعي': 'Collective', 'ángel': 'Angel', 'रोना': 'crying', 'हमराज़': 'Humraj', 'चिलमन': 'Drape', 'औषध': 'Medicine', 'निया': 'Nia', 'תעליעוכופור': 'Go up and go', '中国队大胜': 'Chinese team wins', 'δ
}
def _get_contractions(contractions_dict):
contractions_re = re.compile('(%s)' % '|'.join(contractions_dict.keys()))
return contractions_dict, contractions_re
contractions, contractions_re = _get_contractions(contractions_dict)
 def replace_contractions(text):
def replace(match):
return contractions[match.group(0)]
return contractions_re.sub(replace, text)
 print('REPLACE CONTRACTIONS')
 train_df['question_text'] = train_df['question_text'].apply(lambda x: replace_contractions(x))
test_df['question_text'] = test_df['question_text'].apply(lambda x: replace_contractions(x))
print('FINISH REPLACE CONTRACTIONS')
 list_sentences_train = train_df["question_text"].fillna("_na_" ).values
list_sentences_test = test_df["question_text"].fillna("_na_" ).values
num_words=120000
maxlen=72
print('TOKENIZING')
 tokenizer = Tokenizer(num_words=num_words, char_level=False, lower=False)
 all_text = list(list_sentences_train)+ list(list_sentences_test)
 tokenizer.fit_on_texts(all_text)
 del all_text
gc.collect()
print('FINISH FITTING TOKENIZER')
 train_sequences = tokenizer.texts_to_sequences(list_sentences_train)
 train_sequences = pad_sequences(train_sequences, maxlen=maxlen)
 test_sequences = tokenizer.texts_to_sequences(list_sentences_test)
 test_sequences = pad_sequences(test_sequences, maxlen=maxlen)
 print('FINISH prep sequences')
 del list_sentences_train, list_sentences_test, contractions_dict
gc.collect()
def load_embedding(embedding):
print(f'Loading {embedding} embedding.. ')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def load_word2vec(fname, encoding='utf8', unicode_errors='strict',datatype=np.float32):
embedding_index = dict()
with utils.smart_open(fname)as fin:
header = utils.to_unicode(fin.readline() , encoding=encoding)
 vocab_size, vector_size =(int(x)for x in header.split())
 binary_len = np.dtype(datatype ).itemsize * vector_size
for _ in tqdm(range(vocab_size)) :
word = []
while True:
ch = fin.read(1)
 if ch == b' ':
break
if ch == b'':
raise EOFError("unexpected end of input")
 if ch != b'
':
word.append(ch)
 word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)
 weights = np.fromstring(fin.read(binary_len), dtype=datatype ).astype(datatype)
 embedding_index[word] = weights
return embedding_index
if embedding == 'glove':
EMBEDDING_FILE = f'{FILE_DIR}/embeddings/glove.840B.300d/glove.840B.300d.txt'
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8"))
elif embedding == 'wiki-news':
EMBEDDING_FILE = f'{FILE_DIR}/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8")if len(o)>100)
 elif embedding == 'paragram':
EMBEDDING_FILE = f'{FILE_DIR}/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 elif embedding == 'google-news':
EMBEDDING_FILE = f'{FILE_DIR}/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings_index = load_word2vec(EMBEDDING_FILE)
 return embeddings_index
def build_embedding_matrix(embeddings_index_1, one=True, embedding_name_1='random'):
if one:
embeddings_index_1 = load_embedding(embeddings_index_1)
 wl = WordNetLemmatizer().lemmatize
word_index = tokenizer.word_index
nb_words = min(num_words, len(word_index))
embedding_matrix = np.zeros(( nb_words, 301))
def all_caps(word):
return len(word)> 1 and word.isupper()
if embedding_name_1 == 'google-news':
something_1 = embeddings_index_1.word_vec("something")
 something = np.zeros(( 301,))
something[:300,] = something_1
something[300,] = 0
hit = 0
def embed_word(embedding_matrix,i,word):
embedding_vector_1 = embeddings_index_1.word_vec(word)
 if embedding_vector_1 is not None:
if all_caps(word):
last_value = np.array([1])
 else:
last_value = np.array([0])
 embedding_matrix[i,:300] = embedding_vector_1
embedding_matrix[i,300] = last_value
for word, i in word_index.items() :
if i >= nb_words: continue
if word not in embeddings_index_1.vocab:
embedding_matrix[i] = something
else:
if embeddings_index_1.word_vec(word)is not None:
embed_word(embedding_matrix,i,word)
 hit += 1
else:
if len(word)> 20:
embedding_matrix[i] = something
else:
word2 = wl(wl(word, pos='v'), pos='a')
 if embeddings_index_1.word_vec(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
if len(word)< 3: continue
word2 = word.lower()
if embeddings_index_1.word_vec(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
word2 = word.lower()
word2 = wl(wl(word2, pos='v'), pos='a')
 if embeddings_index_1.word_vec(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
word2 = word.upper()
if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
word2 = word.upper()
word2 = wl(wl(word2, pos='v'), pos='a')
 if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
embedding_matrix[i] = something
else:
something_1 = embeddings_index_1.get("something")
 something = np.zeros(( 301,))
something[:300,] = something_1
something[300,] = 0
hit = 0
def embed_word(embedding_matrix,i,word):
embedding_vector_1 = embeddings_index_1.get(word)
 if embedding_vector_1 is not None:
if all_caps(word):
last_value = np.array([1])
 else:
last_value = np.array([0])
 embedding_matrix[i,:300] = embedding_vector_1
embedding_matrix[i,300] = last_value
for word, i in word_index.items() :
if i >= nb_words: continue
if embeddings_index_1.get(word)is not None:
embed_word(embedding_matrix,i,word)
 hit += 1
else:
if len(word)> 20:
embedding_matrix[i] = something
else:
word2 = wl(wl(word, pos='v'), pos='a')
 if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
if len(word)< 3: continue
word2 = word.lower()
if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
word2 = word.lower()
word2 = wl(wl(word2, pos='v'), pos='a')
 if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
word2 = word.upper()
if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
word2 = word.upper()
word2 = wl(wl(word2, pos='v'), pos='a')
 if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
embedding_matrix[i] = something
del embeddings_index_1
gc.collect()
print("Matched Embeddings: found {} out of total {} words at a rate of {:.2f}%".format(hit, nb_words, hit * 100.0 / nb_words))
return embedding_matrix
def build_concatenate_embedding_matrix_google(embeddings_index_1, embeddings_index_2, one=True, two=True):
if one:
embeddings_index_1 = load_embedding(embeddings_index_1)
 if two:
embeddings_index_2 = load_embedding(embeddings_index_2)
 wl = WordNetLemmatizer().lemmatize
word_index = tokenizer.word_index
nb_words = min(num_words, len(word_index))
embedding_matrix = np.zeros(( nb_words, 401))
something_1 = embeddings_index_1.get("something")
 something = np.zeros(( 401,))
something[:300,] = something_1
something[400,] = 0
def all_caps(word):
return len(word)> 1 and word.isupper()
hit = 0
def embed_word(embedding_matrix,i,word, ori_word):
embedding_vector_1 = embeddings_index_1.get(word)
 if embedding_vector_1 is not None:
if all_caps(word):
last_value = np.array([1])
 else:
last_value = np.array([0])
 embedding_matrix[i,:300] = embedding_vector_1
embedding_matrix[i,400] = last_value
if word in embeddings_index_2.vocab:
embedding_vector_2 = embeddings_index_2.word_vec(ori_word)
 embedding_matrix[i,300:400] = embedding_vector_2
for word, i in word_index.items() :
if i >= nb_words: continue
if embeddings_index_1.get(word)is not None:
embed_word(embedding_matrix,i,word, word)
 hit += 1
else:
if len(word)> 20:
embedding_matrix[i] = something
else:
word2 = wl(wl(word, pos='v'), pos='a')
 if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2, word)
 hit += 1
else:
if len(word)< 3: continue
word2 = word.lower()
if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2, word)
 hit += 1
else:
word2 = word.lower()
word2 = wl(wl(word2, pos='v'), pos='a')
 if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2, word)
 hit += 1
else:
word2 = word.upper()
if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2, word)
 hit += 1
else:
word2 = word.upper()
word2 = wl(wl(word2, pos='v'), pos='a')
 if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2, word)
 hit += 1
else:
embedding_matrix[i] = something
print("Matched Embeddings: found {} out of total {} words at a rate of {:.2f}%".format(hit, nb_words, hit * 100.0 / nb_words))
del embeddings_index_1, embeddings_index_2
gc.collect()
return embedding_matrix
def build_concatenate_embedding_matrix(embeddings_index_1, embeddings_index_2, one=True, two=True):
if one:
embeddings_index_1 = load_embedding(embeddings_index_1)
 if two:
embeddings_index_2 = load_embedding(embeddings_index_2)
 wl = WordNetLemmatizer().lemmatize
word_index = tokenizer.word_index
nb_words = min(num_words, len(word_index))
embedding_matrix = np.zeros(( nb_words, 601))
something_1 = embeddings_index_1.get("something")
 something_2 = embeddings_index_2.get("something")
 something = np.zeros(( 601,))
something[:300,] = something_1
something[300:600,] = something_2
something[600,] = 0
def all_caps(word):
return len(word)> 1 and word.isupper()
hit = 0
def embed_word(embedding_matrix,i,word):
embedding_vector_1 = embeddings_index_1.get(word)
 if embedding_vector_1 is not None:
if all_caps(word):
last_value = np.array([1])
 else:
last_value = np.array([0])
 embedding_matrix[i,:300] = embedding_vector_1
embedding_matrix[i,600] = last_value
embedding_vector_2 = embeddings_index_2.get(word)
 if embedding_vector_2 is not None:
embedding_matrix[i,300:600] = embedding_vector_2
for word, i in word_index.items() :
if i >= nb_words: continue
if embeddings_index_1.get(word)is not None:
embed_word(embedding_matrix,i,word)
 hit += 1
else:
if len(word)> 20:
embedding_matrix[i] = something
else:
word2 = wl(wl(word, pos='v'), pos='a')
 if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
if len(word)< 3: continue
word2 = word.lower()
if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
word2 = word.lower()
word2 = wl(wl(word2, pos='v'), pos='a')
 if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
word2 = word.upper()
if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
word2 = word.upper()
word2 = wl(wl(word2, pos='v'), pos='a')
 if embeddings_index_1.get(word2)is not None:
embed_word(embedding_matrix,i,word2)
 hit += 1
else:
embedding_matrix[i] = something
print("Matched Embeddings: found {} out of total {} words at a rate of {:.2f}%".format(hit, nb_words, hit * 100.0 / nb_words))
del embeddings_index_1, embeddings_index_2
gc.collect()
return embedding_matrix
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def dropout_mask(x, sz, dropout):
return x.new(*sz ).bernoulli_(1-dropout)/(1-dropout)
 class LockedDropout(nn.Module):
def __init__(self, p=0.5):
super().__init__()
self.p=p
def forward(self, x):
if not self.training or not self.p: return x
m = dropout_mask(x.data,(1, x.size(1), x.size(2)) , self.p)
 return Variable(m, requires_grad=False)* x
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def accuracy(output, target, topk=(1,)) :

 with torch.no_grad() :
maxk = max(topk)
 batch_size = target.size(0)
 _, pred = output.topk(maxk, 1, True, True)
 pred = pred.t()
correct = pred.eq(target.view(1, -1 ).expand_as(pred))
res = []
for k in topk:
correct_k = correct[:k].view(-1 ).float().sum(0, keepdim=True)
 res.append(correct_k.mul_(100.0 / batch_size))
return res
def train_model(train_loader, model, criterion, optimizer, epoch, gpu=None, print_freq=100):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
model.train()
epoch_time = time.time()
end = time.time()
for i,(x_batch, x_feats_batch, y_batch)in enumerate(train_loader):
data_time.update(time.time() - end)
 if gpu is not None:
x_batch = x_batch.cuda(gpu, non_blocking=True)
 x_feats_batch= x_feats_batch.cuda(gpu, non_blocking=True)
 y_batch = y_batch.cuda(gpu, non_blocking=True)
 output = model(x_batch, x_feats_batch)
 loss = criterion(output, y_batch)
 acc1 =(y_batch ==(output > 0.5 ).float() ).float()
losses.update(loss.item() , x_batch.size(0))
top1.update(acc1.mean().item() , x_batch.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
 end = time.time()
if i % print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f}({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f}({data_time.avg:.3f})\t'
'Loss {loss.val:.4f}({loss.avg:.4f})\t'
'Acc@1 {top1.avg:.3f}' 
.format(
 epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses, top1=top1))
print('Epoch time: {:.4f}min'.format(( time.time() - epoch_time)/ 60))
return
def train_model_boost(train_loader, model, criterion, optimizer, epoch, gpu=None, print_freq=100):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
model.train()
epoch_time = time.time()
end = time.time()
for i,(x_batch, x_feats_batch, y_batch)in enumerate(train_loader):
data_time.update(time.time() - end)
 if gpu is not None:
x_batch = x_batch.cuda(gpu, non_blocking=True)
 x_feats_batch= x_feats_batch.cuda(gpu, non_blocking=True)
 y_batch = y_batch.cuda(gpu, non_blocking=True)
 output = model(x_batch, x_feats_batch)
 output = torch.nn.Tanh()(output)
 loss = criterion(output, y_batch)
 acc1 =(y_batch ==(output > 0.5 ).float() ).float()
losses.update(loss.item() , x_batch.size(0))
top1.update(acc1.mean().item() , x_batch.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
 end = time.time()
if i % print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f}({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f}({data_time.avg:.3f})\t'
'Loss {loss.val:.4f}({loss.avg:.4f})\t'
'Acc@1 {top1.avg:.3f}' 
.format(
 epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses, top1=top1))
print('Epoch time: {:.4f}min'.format(( time.time() - epoch_time)/ 60))
return
def test_model(loader, model, gpu=None, return_true=False):
model.eval()
outputs = []
with torch.no_grad() :
end = time.time()
for i,(x_batch, x_feats_batch)in enumerate(loader):
if gpu is not None:
x_batch = x_batch.cuda(gpu, non_blocking=True)
 x_feats_batch = x_feats_batch.cuda(gpu, non_blocking=True)
 output = model(x_batch, x_feats_batch)
 y_pred = output.detach()
outputs.append(sigmoid(y_pred.cpu().numpy())[:, 0])
 outputs = np.concatenate(outputs)
 return outputs
def test_model_boost(loader, model, gpu=None, return_true=False):
model.eval()
outputs = []
with torch.no_grad() :
end = time.time()
for i,(x_batch, x_feats_batch)in enumerate(loader):
if gpu is not None:
x_batch = x_batch.cuda(gpu, non_blocking=True)
 x_feats_batch = x_feats_batch.cuda(gpu, non_blocking=True)
 output = model(x_batch, x_feats_batch)
 output = torch.nn.Tanh()(output)
 y_pred = output.detach()
outputs.append(y_pred.cpu().numpy())
 outputs = np.concatenate(outputs)
 return outputs
class ModelRNN(nn.Module):
def __init__(self, max_features, embedding_matrix, hidden_size, linear1_in, linear1_out, lockout, dropout):
super(ModelRNN, self ).__init__()
self.max_features = max_features
self.embedding_matrix = embedding_matrix
self.hidden_size = hidden_size
self.linear1_in = linear1_in
self.linear1_out = linear1_out
self.locked_dropout = LockedDropout(lockout)
 self.dropout = dropout
embed_size = self.embedding_matrix.shape[1]
self.embedding = nn.Embedding(self.max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size*2, bidirectional=True, batch_first=True)
 self.linear = nn.Linear(self.linear1_in, self.linear1_out)
 self.relu = nn.ReLU()
self.out = nn.Linear(self.linear1_out, 1)
 def forward(self, x, x_feats):
h_embedding = self.embedding(x)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 last_state = h_gru[:, -1, :]
conc = torch.cat(( avg_pool, max_pool, last_state, x_feats), 1)
 conc = self.relu(self.linear(conc))
out = self.out(conc)
 return out
class ModelRNN_fc2(nn.Module):
def __init__(self, max_features, embedding_matrix, hidden_size, linear1_in, linear1_out, linear2_out, lockout, dropout):
super(ModelRNN_fc2, self ).__init__()
self.max_features = max_features
self.embedding_matrix = embedding_matrix
self.hidden_size = hidden_size
self.linear1_in = linear1_in
self.linear1_out = linear1_out
self.linear2_out = linear2_out
self.locked_dropout = LockedDropout(lockout)
 self.dropout = dropout
embed_size = self.embedding_matrix.shape[1]
self.embedding = nn.Embedding(self.max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size*2, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(self.linear1_in, self.linear1_out)
 self.relu1 = nn.ReLU()
self.linear2 = nn.Linear(self.linear1_out, self.linear2_out)
 self.relu2 = nn.ReLU()
self.dropout = nn.Dropout(self.dropout)
 self.out = nn.Linear(self.linear2_out, 1)
 def forward(self, x, x_feats):
h_embedding = self.embedding(x)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 last_state = h_gru[:, -1, :]
conc = torch.cat(( avg_pool, max_pool, last_state, x_feats), 1)
 conc = self.relu1(self.linear1(conc))
conc = self.dropout(conc)
 conc = self.relu2(self.linear2(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out
class DPCNNTextClassifier(nn.Module):

 def __init__(self, max_features, embedding_matrix, hidden_size, input_dropout_p):
super(DPCNNTextClassifier, self ).__init__()
self.max_features = max_features
self.embedding_matrix = embedding_matrix
self.hidden_size = hidden_size
self.input_dropout_p = input_dropout_p
embed_size = self.embedding_matrix.shape[1]
self.embedding = nn.Embedding(self.max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.input_dropout = nn.Dropout(p=input_dropout_p)
 self.channel_size = 150
self.conv_region_embedding = nn.Conv2d(1, self.channel_size,(3, embed_size), stride=1)
 self.conv3 = nn.Conv2d(self.channel_size, self.channel_size,(3, 1), stride=1)
 self.pooling = nn.MaxPool2d(kernel_size=(3, 1), stride=2)
 self.padding_conv = nn.ZeroPad2d(( 0, 0, 1, 1))
self.padding_pool = nn.ZeroPad2d(( 0, 0, 0, 1))
self.act_fun = nn.ReLU()
self.linear_out = nn.Linear(self.channel_size, 1)
 def forward(self, input_var, lengths=None):
embeded = self.embedding(input_var)
 embeded = self.input_dropout(embeded)
 batch, width, height = embeded.shape
embeded = embeded.view(( batch, 1, width, height))
x = self.conv_region_embedding(embeded)
 x = self.padding_conv(x)
 x = self.act_fun(x)
 x = self.conv3(x)
 x = self.padding_conv(x)
 x = self.act_fun(x)
 x = self.conv3(x)
 while x.size() [-2] >= 2:
x = self._block(x)
 x = x.view(batch, self.channel_size)
 x = self.linear_out(x)
 return x
def _block(self, x):
x = self.padding_pool(x)
 px = self.pooling(x)
 x = self.padding_conv(px)
 x = F.relu(x)
 x = self.conv3(x)
 x = self.padding_conv(x)
 x = F.relu(x)
 x = self.conv3(x)
 x = x + px
return x
hidden_size = 90
gru_len = hidden_size
Routings = 4
Num_capsule = 5
Dim_capsule = 5
dropout_p = 0.25
rate_drop_dense = 0.28
LR = 0.001
T_epsilon = 1e-7
num_classes = 30
class Embed_Layer(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=601):
super(Embed_Layer, self ).__init__()
self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)
 if use_pretrained_embedding:
self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))
def forward(self, x, dropout_p=0.25):
return nn.Dropout(p=dropout_p )(self.encoder(x))
class GRU_Layer(nn.Module):
def __init__(self):
super(GRU_Layer, self ).__init__()
self.gru = nn.GRU(input_size=300,
hidden_size=gru_len,
bidirectional=True)

def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.gru(x)
 class Caps_Layer(nn.Module):
def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \
routings=Routings, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Caps_Layer, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = self.squash
else:
self.activation = nn.ReLU(inplace=True)
 if self.share_weights:
self.W = nn.Parameter(
 nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))
 else:
self.W = nn.Parameter(
 t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))
def forward(self, x):
if self.share_weights:
u_hat_vecs = t.matmul(x, self.W)
 else:
print('add later')
 batch_size = x.size(0)
 input_num_capsule = x.size(1)
 u_hat_vecs = u_hat_vecs.view(( batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)
 b = t.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = b.permute(0, 2, 1)
 c = F.softmax(b, dim=2)
 c = c.permute(0, 2, 1)
 b = b.permute(0, 2, 1)
 outputs = self.activation(t.einsum('bij,bijk->bik',(c, u_hat_vecs)))
 if i < self.routings - 1:
b = t.einsum('bik,bijk->bij',(outputs, u_hat_vecs))
return outputs
def squash(self, x, axis=-1):
s_squared_norm =(x ** 2 ).sum(axis, keepdim=True)
 scale = t.sqrt(s_squared_norm + T_epsilon)
 return x / scale
class Capsule_Main(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None):
super(Capsule_Main, self ).__init__()
self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)
 self.gru_layer = GRU_Layer()
self.gru_layer.init_weights()
self.caps_layer = Caps_Layer()
self.dense_layer = Dense_Layer()
def forward(self, content):
content1 = self.embed_layer(content)
 content2, _ = self.gru_layer(
 content1)
 content3 = self.caps_layer(content2)
 output = self.dense_layer(content3)
 return output
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class CapsNet(nn.Module):
def __init__(self, max_features, embedding_matrix, hidden_size, linear1_in):
super(CapsNet, self ).__init__()
self.max_features = max_features
self.embedding_matrix = embedding_matrix
self.hidden_size = hidden_size
self.linear1_in = linear1_in
fc_layer = 16
fc_layer1 = 16
embed_size = self.embedding_matrix.shape[1]
self.embedding = nn.Embedding(self.max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.bn = nn.BatchNorm1d(16, momentum=0.5)
 self.linear = nn.Linear(self.linear1_in, fc_layer1)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.fc = nn.Linear(fc_layer**2,fc_layer)
 self.out = nn.Linear(fc_layer, 1)
 self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)
 self.caps_layer = Caps_Layer()
def forward(self, x, x_feats):
h_embedding = self.embedding(x)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 content3 = self.caps_layer(h_gru)
 content3 = self.dropout(content3)
 batch_size = content3.size(0)
 content3 = content3.view(batch_size, -1)
 content3 = self.relu(self.lincaps(content3))
avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 last_state = h_gru[:, -1, :]
conc = torch.cat(( content3, avg_pool, max_pool, last_state, x_feats), 1)
 conc = self.relu(self.linear(conc))
conc = self.bn(conc)
 conc = self.dropout(conc)
 out = self.out(conc)
 return out
gpu = 0
batch_size = 512
start_epoch = 0
np.save('train_sequences.npy',train_sequences)
 np.save('training_labels.npy',training_labels)
 np.save('train_features.npy',train_features)
 np.save('test_sequences.npy',test_sequences)
 np.save('test_features.npy',test_features)
 print('PREPARE FOR PYTORCH')
 x_train = torch.tensor(train_sequences, dtype=torch.long ).cuda()
y_train = torch.tensor(training_labels[:, np.newaxis], dtype=torch.float32 ).cuda()
x_train_feats = torch.tensor(train_features, dtype=torch.float32 ).cuda()
print('FINISH TRAIN PREMINILARY FOR PYTORCH')
 x_train_data = torch.utils.data.TensorDataset(x_train, x_train_feats, y_train)
 train = torch.utils.data.DataLoader(x_train_data, batch_size=batch_size, shuffle=True)
 del x_train, x_train_feats, x_train_data, y_train
gc.collect()
print('FINISH TRAIN FOR PYTORCH')
 x_test = torch.tensor(test_sequences, dtype=torch.long ).cuda()
x_test_feats = torch.tensor(test_features, dtype=torch.float32 ).cuda()
x_test_data = torch.utils.data.TensorDataset(x_test, x_test_feats)
 test = torch.utils.data.DataLoader(x_test_data, batch_size=batch_size, shuffle=False)
 del train_sequences, train_features, test_sequences, test_features
gc.collect()
del x_test, x_test_feats, x_test_data
gc.collect()
embed_google_index = load_embedding('google-news')
 embedding_matrix = build_embedding_matrix(embed_google_index, one=False)
 rnn_kwargs = {
'max_features': num_words,
'embedding_matrix': embedding_matrix,
'hidden_size': 90,
'linear1_in': 1082,
'linear1_out': 70,
'linear2_out':16,
'lockout': 0.1,
'dropout': 0.1,
}
del embedding_matrix
gc.collect()
epochs = 4
model = ModelRNN_fc2(**rnn_kwargs)
 del rnn_kwargs
gc.collect()
model.cuda()
criterion = torch.nn.BCEWithLogitsLoss(reduction="sum")
 lr=0.003
for epoch in range(start_epoch, epochs):
optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 train_model(train, model, criterion, optimizer, epoch, gpu=gpu)
 lr = lr*0.8
preds = np.zeros(( len(test_idx)))
 preds = test_model(test, model, gpu=gpu)
 preds = preds.ravel()
if split>0:
best_score, _ = f1_smart(testing_labels, preds)
 print('RNN2-googlenews :', best_score)
 np.save('preds_googlenews.npy', preds)
 del model, criterion, optimizer, preds
gc.collect()
embedding_matrix = build_concatenate_embedding_matrix('paragram',embed_google_index, two=False)
 del embed_google_index
gc.collect()
rnn_kwargs = {
'max_features': num_words,
'embedding_matrix': embedding_matrix,
'hidden_size': hidden_size,
'linear1_in': 543,
}
del embedding_matrix
gc.collect()
start_epoch = 0
model = CapsNet(**rnn_kwargs)
 del rnn_kwargs
gc.collect()
model.cuda()
criterion = torch.nn.BCEWithLogitsLoss(reduction="sum")
 epochs = 3
lr=0.003
for epoch in range(start_epoch, epochs):
optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 train_model(train, model, criterion, optimizer, epoch, gpu=gpu)
 lr = lr*0.8
preds = np.zeros(( len(test_idx)))
 preds = test_model(test, model, gpu=gpu)
 preds = preds.ravel()
if split>0:
best_score, _ = f1_smart(testing_labels, preds)
 print('RNN2-paragram_google :', best_score)
 np.save('preds_paragram_google.npy', preds)
 del train, test
gc.collect()
train_sequences = np.load('train_sequences.npy')
 train_features = np.load('train_features.npy')
 test_sequences = np.load('test_sequences.npy')
 test_features = np.load('test_features.npy')
 oof_pred_LR = np.load('oof_pred_LR.npy')
 training_labels_boost = training_labels - oof_pred_LR
del oof_pred_LR
gc.collect()
print('PREPARE FOR PYTORCH')
 x_train = torch.tensor(train_sequences, dtype=torch.long ).cuda()
y_train = torch.tensor(training_labels_boost[:, np.newaxis], dtype=torch.float32 ).cuda()
x_train_feats = torch.tensor(train_features, dtype=torch.float32 ).cuda()
print('FINISH TRAIN PREMINILARY FOR PYTORCH')
 x_train_data = torch.utils.data.TensorDataset(x_train, x_train_feats, y_train)
 train = torch.utils.data.DataLoader(x_train_data, batch_size=batch_size, shuffle=True)
 del x_train, x_train_feats, x_train_data, y_train
gc.collect()
print('FINISH TRAIN FOR PYTORCH')
 x_test = torch.tensor(test_sequences, dtype=torch.long ).cuda()
x_test_feats = torch.tensor(test_features, dtype=torch.float32 ).cuda()
x_test_data = torch.utils.data.TensorDataset(x_test, x_test_feats)
 test = torch.utils.data.DataLoader(x_test_data, batch_size=batch_size, shuffle=False)
 del train_sequences, train_features, test_sequences, test_features, training_labels_boost
gc.collect()
embed_glove_index = load_embedding('glove')
 embedding_matrix = build_embedding_matrix(embed_glove_index, one=False)
 del model
gc.collect()
rnn_kwargs = {
'max_features': num_words,
'embedding_matrix': embedding_matrix,
'hidden_size': 90,
'linear1_in': 1082,
'linear1_out': 70,
'lockout': 0.1,
'dropout': 0.1,
}
epochs = 3
model = ModelRNN(**rnn_kwargs)
 del embedding_matrix, rnn_kwargs
gc.collect()
model.cuda()
criterion = torch.nn.MSELoss()
lr=0.003
for epoch in range(start_epoch, epochs):
optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 train_model_boost(train, model, criterion, optimizer, epoch, gpu=gpu)
 lr = lr*0.8
preds = np.zeros(( len(test_idx)))
 preds = test_model_boost(test, model, gpu=gpu)
 preds = preds.ravel()
test_pred_LR = np.load('test_pred_LR.npy')
 preds = preds + test_pred_LR
del test_pred_LR
gc.collect()
if split>0:
best_score, _ = f1_smart(testing_labels, preds)
 print('RNN2-glove :', best_score)
 np.save('preds_glove.npy', preds)
 del model, criterion, optimizer, preds, train
gc.collect()
train_sequences = np.load('train_sequences.npy')
 train_features = np.load('train_features.npy')
 print('PREPARE FOR PYTORCH')
 x_train = torch.tensor(train_sequences, dtype=torch.long ).cuda()
y_train = torch.tensor(training_labels[:, np.newaxis], dtype=torch.float32 ).cuda()
x_train_feats = torch.tensor(train_features, dtype=torch.float32 ).cuda()
print('FINISH TRAIN PREMINILARY FOR PYTORCH')
 x_train_data = torch.utils.data.TensorDataset(x_train, x_train_feats, y_train)
 train = torch.utils.data.DataLoader(x_train_data, batch_size=batch_size, shuffle=True)
 del x_train, x_train_feats, x_train_data, y_train
gc.collect()
del train_sequences, train_features
gc.collect()
embedding_matrix = build_concatenate_embedding_matrix(embed_glove_index, 'wiki-news', one=False)
 del embed_glove_index
gc.collect()
rnn_kwargs = {
'max_features': num_words,
'embedding_matrix': embedding_matrix,
'hidden_size': hidden_size,
'linear1_in': 543,
}
epochs = 3
model = CapsNet(**rnn_kwargs)
 del embedding_matrix, rnn_kwargs
gc.collect()
model.cuda()
criterion = torch.nn.BCEWithLogitsLoss(reduction="sum")
 lr=0.003
for epoch in range(start_epoch, epochs):
optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 train_model(train, model, criterion, optimizer, epoch, gpu=gpu)
 lr=lr*0.8
preds = np.zeros(( len(test_idx)))
 preds = test_model(test, model, gpu=gpu)
 preds = preds.ravel()
if split>0:
best_score, _ = f1_smart(testing_labels, preds)
 print('RNN-glove+fasttext :', best_score)
 np.save('preds_glove_fasttext.npy', preds)
 del model, criterion, optimizer, preds
gc.collect()
x_train = {'sparse_data_one': scipy.sparse.load_npz('train_char_features.npz'),
'sparse_data_two': scipy.sparse.load_npz('train_name_bi.npz'),
'sparse_data_three': scipy.sparse.load_npz('train_word_features.npz'),
'feats': np.load('train_sparse_feats.npy'),
}
def sparseNN() :
sparse_data_one = Input(shape=[x_train["sparse_data_one"].shape[1]],
dtype = 'float32', sparse = True, name='sparse_data_one')
 sparse_data_two = Input(shape=[x_train["sparse_data_two"].shape[1]],
dtype = 'float32', sparse = True, name='sparse_data_two')
 sparse_data_three = Input(shape=[x_train["sparse_data_three"].shape[1]],
dtype = 'float32', sparse = True, name='sparse_data_three')
 feats = Input(shape=[6], name="feats")
 x_one = Dense(200 , kernel_initializer=he_uniform(seed=0))(sparse_data_one)
 x_one = PReLU()(x_one)
 x_two = Dense(200 , kernel_initializer=he_uniform(seed=0))(sparse_data_two)
 x_two = PReLU()(x_two)
 x_three = Dense(200 , kernel_initializer=he_uniform(seed=0))(sparse_data_three)
 x_three = PReLU()(x_three)
 x = concatenate([x_one, x_two, x_three, feats])
 x = Dense(200 , kernel_initializer=he_uniform(seed=0))(x)
 x = PReLU()(x)
 x = Dense(100 , kernel_initializer=he_uniform(seed=0))(x)
 x = PReLU()(x)
 x= Dense(1, activation='sigmoid' )(x)
 model = Model([sparse_data_one, sparse_data_two, sparse_data_three, feats], x)
 optimizer = Adam (.0005)
 model.compile(loss="binary_crossentropy", optimizer=optimizer)
 return model
BATCH_SIZE = 1024
epochs = 1
sparse_nn = sparseNN()
sparse_nn.fit(x_train, training_labels, batch_size=BATCH_SIZE, epochs=1, verbose=1)
 del x_train, training_labels
gc.collect()
x_test = {'sparse_data_one': scipy.sparse.load_npz('test_char_features.npz'),
'sparse_data_two': scipy.sparse.load_npz('test_name_bi.npz'),
'sparse_data_three': scipy.sparse.load_npz('test_word_features.npz'),
'feats': np.load('test_sparse_feats.npy'),
}
preds = sparse_nn.predict(x_test)[:,0]
np.save('preds_sparsenn.npy', preds)
 def do_sparses(dfs):
def is_chinese(x):
if re.search(u'[\u4e00-\u9fff]', x):
return 1
else:
return 0
def get_sentiment(df):
sid = SIA()
df['nltk'] = df['question_text'].apply(lambda x: sid.polarity_scores(x))
df['neg'] = df['nltk'].apply(lambda x: x['neg'])
 df['neu'] = df['nltk'].apply(lambda x: x['neu'])
 df['pos'] = df['nltk'].apply(lambda x: x['pos'])
 df['compound'] = df['nltk'].apply(lambda x: x['compound'])
 df['chinese']= df['question_text'].apply(lambda x: is_chinese(x))
def create_count_features(df_data):
def lg(text):
text = [x for x in text.split() if x!='']
return len(text)
 df_data['nb_words_description'] = df_data['question_text'].apply(lg ).astype(np.uint16)
 return df_data
train = pd.read_csv(dfs[0])
 test = pd.read_csv(dfs[1])
 train_target = train['target'].values
np.save('train_target.npy',train_target)
 print('AHA IM HERE fitting new tfidf !')
 TOKENIZER = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s):
return TOKENIZER.sub(r' \1 ', s ).split()
def lemmatize_sentence(sentence):
tokens=nltk.word_tokenize(sentence)
 lemmatizer = WordNetLemmatizer()
lem = [lemmatizer.lemmatize(t)for t in tokens]
return " ".join(lem)
 train['question_text_lemma'] = train['question_text'].apply(lambda x: lemmatize_sentence(x))
tfidf_vectorizer = TfidfVectorizer(
 ngram_range=(1,3),
tokenizer=tokenize,
min_df=3,
max_df=0.9,
strip_accents='unicode',
use_idf=True,
smooth_idf=True,
sublinear_tf=True,
max_features = 100000
).fit(train['question_text_lemma'])
 print('FINISH fitting new tfidf !')
 X_tfidf = tfidf_vectorizer.transform(train['question_text_lemma'])
 scipy.sparse.save_npz('X_tfidf.npz', X_tfidf)
 del X_tfidf, train['question_text_lemma']
gc.collect()
print('FINISH transforming new tfidf in train!')
 test['question_text_lemma'] = test['question_text'].apply(lambda x: lemmatize_sentence(x))
X_tfidf_test = tfidf_vectorizer.transform(test['question_text_lemma'])
 scipy.sparse.save_npz('X_tfidf_test.npz', X_tfidf_test)
 del X_tfidf_test, tfidf_vectorizer, test['question_text_lemma']
gc.collect()
char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
stop_words='english',
ngram_range=(2, 4),
max_features=30000)
 char_vectorizer.fit(train['question_text'])
 print('Char TFIDF 1/3')
 train_char_features = char_vectorizer.transform(train['question_text'])
 scipy.sparse.save_npz('train_char_features.npz', train_char_features)
 del train_char_features
gc.collect()
print('Char TFIDF 2/3')
 test_char_features = char_vectorizer.transform(test['question_text'])
 scipy.sparse.save_npz('test_char_features.npz', test_char_features)
 del char_vectorizer, test_char_features
gc.collect()
print('Char TFIDF 3/3')
 word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
max_features=30000)
 word_vectorizer.fit(train['question_text'])
 print('Word TFIDF 1/3')
 train_word_features = word_vectorizer.transform(train['question_text'])
 scipy.sparse.save_npz('train_word_features.npz', train_word_features)
 del train_word_features
gc.collect()
print('Word TFIDF 2/3')
 test_word_features = word_vectorizer.transform(test['question_text'])
 scipy.sparse.save_npz('test_word_features.npz', test_word_features)
 del word_vectorizer, test_word_features
gc.collect()
print('Word TFIDF 3/3')
 get_sentiment(train)
 train = create_count_features(train)
 np.save('train_sparse_feats.npy',train[['chinese','neg','neu','pos','compound','nb_words_description']].values)
 train.drop(['chinese','neg','neu','pos','compound','nb_words_description'],axis=1, inplace=True)
 get_sentiment(test)
 test = create_count_features(test)
 np.save('test_sparse_feats.npy',test[['chinese','neg','neu','pos','compound','nb_words_description']].values)
 test.drop(['chinese','neg','neu','pos','compound','nb_words_description'],axis=1, inplace=True)
 print('START LOADING TFIDF')
 train_target = np.load('train_target.npy')
 n_folds = 5
test_pred_LR = 0
oof_pred_LR = np.zeros([train_target.shape[0],])
 X_tfidf = scipy.sparse.load_npz('X_tfidf.npz')
 skf = list(StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed ).split(X_tfidf, train_target))
del train_target, X_tfidf
gc.collect()
for i,(train_index, val_index)in tqdm(enumerate(skf)) :
X_tfidf = scipy.sparse.load_npz('X_tfidf.npz')
 x_train, x_val = X_tfidf[list(train_index)], X_tfidf[list(val_index)]
del X_tfidf
gc.collect()
train_target = np.load('train_target.npy')
 y_train, y_val = train_target[train_index], train_target[val_index]
del train_target
gc.collect()
classifier = Ridge()
classifier.fit(x_train, y_train)
 del x_train, y_train
gc.collect()
val_preds = classifier.predict(x_val)
 X_tfidf_test = scipy.sparse.load_npz('X_tfidf_test.npz')
 preds = classifier.predict(X_tfidf_test)
 del X_tfidf_test,x_val
gc.collect()
test_pred_LR += 0.2*preds
oof_pred_LR[val_index] = val_preds
print(f1_score(y_val, val_preds > 0.26))
del classifier, preds, val_preds
gc.collect()
np.save('oof_pred_LR.npy',oof_pred_LR)
 del oof_pred_LR
gc.collect()
np.save('test_pred_LR.npy',test_pred_LR)
 del test_pred_LR
gc.collect()
print('END Parallel')
 wordnet_lemmatizer = WordNetLemmatizer()
def word_count(text, dc):
text = set(text.split(' '))
for w in text:
dc[w]+=1
def remove_low_freq(text, dc):
return ' '.join([w for w in text.split() if w in dc])
 stop_words = set(stopwords.words('english'))
def create_bigrams(text):
text = np.unique([ wordnet_lemmatizer.lemmatize(w)for w in text.split() if w not in stop_words ])
 lst_bi = []
for combo in combinations(text, 2):
cb1=combo[0]+combo[1]
cb2=combo[1]+combo[0]
in_dict=False
if cb1 in word_count_dict_one:
new_word = cb1
in_dict=True
if cb2 in word_count_dict_one:
new_word = cb2
in_dict=True
if not in_dict:
new_word = combo[0]+'___'+combo[1]
if len(cb1)>=0:
lst_bi.append(new_word)
 return ' '.join(lst_bi)
 def create_bigrams_df(df):
return df.apply(create_bigrams)
 cores = 2
max_text_length=60
min_df_one=5
min_df_bi=5
def parallelize_dataframe(df, func):
df_split = np.array_split(df, cores)
 pool = Pool(cores)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df
word_count_dict_one = defaultdict(np.uint32)
 train['question_text'].apply(lambda x : word_count(x, word_count_dict_one))
rare_words = [key for key in word_count_dict_one if word_count_dict_one[key]<min_df_one ]
for key in rare_words :
word_count_dict_one.pop(key, None)
 train['question_text'] = train['question_text'].apply(lambda x : remove_low_freq(x, word_count_dict_one))
word_count_dict_one=dict(word_count_dict_one)
 start_time = time.time()
word_count_dict_bi=defaultdict(np.uint32)
 def word_count_bi(text):
text = text.split(' ')
 for w in text:
word_count_dict_bi[w]+=1
train['question_textbi'] = train['question_text'].apply(lambda x : ' '.join(x.split() [5:]))
test['question_textbi'] = test['question_text'].apply(lambda x : ' '.join(x.split() [5:]))
train['name_bi'] = create_bigrams_df(train['question_textbi'])
 test['name_bi'] = create_bigrams_df(test['question_textbi'])
 train.drop('question_textbi',axis=1, inplace=True)
 test.drop('question_textbi',axis=1, inplace=True)
 gc.collect()
train['name_bi'].apply(word_count_bi)
 rare_words = [key for key in word_count_dict_bi if word_count_dict_bi[key]<min_df_bi ]
for key in rare_words :
word_count_dict_bi.pop(key, None)
 train['name_bi'] = train['name_bi'].apply(lambda x : remove_low_freq(x, word_count_dict_bi))
test['name_bi'] = test['name_bi'].apply(lambda x : remove_low_freq(x, word_count_dict_bi))
print('[{}] Finished CREATING BIGRAMS...'.format(time.time() - start_time))
start_time = time.time()
word_count_dict_bi = dict(word_count_dict_bi)
 vocabulary_one = word_count_dict_one.copy()
vocabulary_bi = word_count_dict_bi.copy()
for dc in [vocabulary_one, vocabulary_bi]:
cpt=0
for key in dc:
dc[key]=cpt
cpt+=1
print('[{}] Finished CREATING VOCABULARY...'.format(time.time() - start_time))
def tokenize(text):
return [w for w in text.split() ]
start_time = time.time()
vect_item_one = CountVectorizer(vocabulary= vocabulary_one, dtype=np.uint8,
tokenizer=tokenize, binary=True)
 train_item_one = vect_item_one.fit_transform(train['question_text'])
 scipy.sparse.save_npz('train_item_one.npz', train_item_one)
 del train_item_one, vocabulary_one
gc.collect()
test_item_one = vect_item_one.transform(test['question_text'])
 scipy.sparse.save_npz('test_item_one.npz', test_item_one)
 del test_item_one
gc.collect()
print('[{}] Finished Vectorizing Onegram Item Description'.format(time.time() - start_time))
start_time = time.time()
vect_name_bi = CountVectorizer(vocabulary= vocabulary_bi, dtype=np.uint8,
tokenizer=tokenize, binary=True)
 train_name_bi = vect_name_bi.fit_transform(train['name_bi'])
 scipy.sparse.save_npz('train_name_bi.npz', train_name_bi)
 del train['name_bi'], train_name_bi, vocabulary_bi
gc.collect()
print('[{}] Finished Vectorizing BiGram Name'.format(time.time() - start_time))
test_name_bi = vect_name_bi.transform(test['name_bi'])
 scipy.sparse.save_npz('test_name_bi.npz', test_name_bi)
 del test_name_bi, word_count_dict_one, rare_words
print('END Parallel')
 df_two = [train, test, True]
test_idx = list(test.qid.values)
 del train, test
gc.collect()
pool = Pool(1)
 pool.apply_async(do_sparses, [[f'{FILE_DIR}/train.csv', f'{FILE_DIR}/test.csv']])
 do_rnns(df_two)
<load_pretrained>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
models_predictions = defaultdict(list)
 models_predictions['RNN2-googlenews'] += list(np.load('preds_googlenews.npy'))
models_predictions['RNN2-paragram_google'] += list(np.load('preds_paragram_google.npy'))
models_predictions['RNN2-glove'] += list(np.load('preds_glove.npy'))
models_predictions['RNN2-glove_fasttext'] += list(np.load('preds_glove_fasttext.npy'))
models_predictions['RNN2-sparsenn'] += list(np.load('preds_sparsenn.npy'))<save_to_csv>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
submission_preds_df = pd.DataFrame(models_predictions)
 submission_preds_df.corr()
mysubmission=pd.DataFrame()
mysubmission['qid'] = test_idx
preds = submission_preds_df.mean(axis=1)
 mysubmission['prediction'] = preds > 0.44190952125936747
mysubmission.to_csv('submission.csv', index=False)
 print(mysubmission.shape )<set_options>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
os.environ['OMP_NUM_THREADS'] = '2'
pd.set_option('display.max_colwidth', -1)
<load_from_csv>
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
train_df = pd.read_csv(".. /input/train.csv",usecols=['question_text','target'])
 test_df = pd.read_csv(".. /input/test.csv")
 train_df, val_df = train_test_split(train_df, test_size=0.001, random_state = 200 )<count_values>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
train_df['target'].value_counts()<count_values>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
val_df['target'].value_counts(normalize = True )<feature_engineering>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
<prepare_x_and_y>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
X_train = train_df['transform'].values
X_val = val_df['transform'].values
y_train = train_df['target'].values
y_val = val_df['target'].values
X_test = test_df['transform'].values 
<string_transform>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
%%time
maxlen = 50
tokenizer = text.Tokenizer(filters='\t
')
 tokenizer.fit_on_texts(list(X_train))
',oov_token=set(list(oov[0].values)))
 X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen, padding='post',truncating='post')
 x_val = sequence.pad_sequences(X_val, maxlen=maxlen, padding='post', truncating='post')
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen, padding='post', truncating='post' )<define_variables>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
word_index = tokenizer.word_index
embedding_matrix1 = np.zeros(( max(list(word_index.values())) + 1, 300), dtype = 'float32')
 embedding_matrix2 = np.zeros(( max(list(word_index.values())) + 1, 300), dtype = 'float32')
 len(word_index )<feature_engineering>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
f = open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 for line in tqdm(f):
values = line.split(" ")
 if(len(values)< 200):
print("a")
 word = values[0]
if word not in word_index:
continue
embedding_matrix1[word_index[word]] = np.asarray(values[1:], dtype='float32')
 f.close()<feature_engineering>
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.25*df1['Confidence'] + 0.75*df2['Confidence']
df.head()
<feature_engineering><EOS> 
df.to_csv('submission.csv', index=False )