Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<define_variables>
from pathlib import Path
import numpy as np
import pandas as pd
import lightgbm as lgb
import os
from sklearn.model_selection import KFold
from tqdm.notebook import tqdm
import seaborn as sbn
from datetime import datetime, timedelta
from sklearn import datasets, linear_model
import matplotlib.pyplot as plt
import statsmodels.api as sm
import random
from scipy.stats import norm
MAX_SEQUENCE_LENGTH = 60
MAX_WORDS = 45000
EMBEDDINGS_LOADED_DIMENSIONS = 300<load_from_csv>
pd.options.mode.chained_assignment = None
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv" )<define_variables>
path = Path('/kaggle/input/osic-pulmonary-fibrosis-progression')
 assert path.exists()
BATCH_SIZE = 512
Q_FRACTION = 1
questions = df_train.sample(frac=Q_FRACTION)
 question_texts = questions["question_text"].values
question_targets = questions["target"].values
kaggle_texts = df_test["question_text"].fillna("_na_" ).values
print(f"Working on {len(questions)} questions" )<load_pretrained>
TRAIN_TYPES={"Patient": "category",
"Weeks": "int16", "FVC": "int32", 'Percent': 'float32', "Age": "uint8",
"Sex": "category", "SmokingStatus": "category" }
SUBMISSION_TYPES={"Patient_Week": "category", "FVC": "int32", "Confidence": "int16"}
def read_data(path):
train_df = pd.read_csv(path/'train.csv', dtype = TRAIN_TYPES)
 test_df = pd.read_csv(path/'test.csv', dtype = TRAIN_TYPES)
 submission_df = pd.read_csv(path/'sample_submission.csv', dtype = SUBMISSION_TYPES)
 train_df.drop_duplicates(keep='first', inplace=True, subset=['Patient','Weeks'])
 return train_df, test_df, submission_df
def load_embeddings(file):
embeddings = {}
with open(file, encoding="utf8", errors='ignore')as f:
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings = dict(get_coefs(*line.split(" ")) for(i, line)in enumerate(tqdm(f)))
 print('Found %s word vectors.' % len(embeddings))
return embeddings<categorify>
train_df, test_df, submission_df = read_data(path )
def create_embedding_weights(tokenizer, embeddings, dimensions):
not_embedded = defaultdict(int)
 word_index = tokenizer.word_index
words_count = min(len(word_index), MAX_WORDS)
 embeddings_matrix = np.zeros(( words_count, dimensions))
for word, i in word_index.items() :
if i >= MAX_WORDS:
continue
if word not in embeddings:
continue
embedding_vector = embeddings[word]
if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector
return embeddings_matrix<compute_train_metric>
def prepare_submission(df, test_df):
df['Patient'] = df['Patient_Week'].apply(lambda x:x.split('_')[0])
 df['Weeks'] = df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
df = df[['Patient','Weeks','Confidence','Patient_Week']]
df = df.merge(test_df.drop('Weeks', axis=1 ).copy() , on=['Patient'])
 return df
THRESHOLD = 0.35
class EpochMetricsCallback(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.f1s = []
self.precisions = []
self.recalls = []
def on_epoch_end(self, epoch, logs={}):
predictions = self.model.predict(self.validation_data[0])
 predictions =(predictions > THRESHOLD ).astype(int)
 predictions = np.asarray(predictions)
 targets = self.validation_data[1]
f1 = metrics.f1_score(targets, predictions)
 precision = metrics.precision_score(targets, predictions)
 recall = metrics.recall_score(targets, predictions)
 print(" - F1 score: {0:.4f}, Precision: {1:.4f}, Recall: {2:.4f}" 
.format(f1, precision, recall))
self.f1s.append(f1)
 self.precisions.append(precision)
 self.recalls.append(recall)
 return
def display_model_history(history):
data = pd.DataFrame(data={'Train': history.history['loss'], 'Test': history.history['val_loss']})
 ax = sns.lineplot(data=data, palette="pastel", linewidth=2.5, dashes=False)
 ax.set(xlabel='Epoch', ylabel='Loss', title='Loss')
 ax.xaxis.set_major_locator(MaxNLocator(integer=True))
sns.despine()
plt.show()
def display_model_epoch_metrics(epoch_callback):
data = pd.DataFrame(data = {
'F1': epoch_callback.f1s,
'Precision': epoch_callback.precisions,
'Recall': epoch_callback.recalls})
 ax = sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)
 ax.set(xlabel='Epoch', title='Epoch metrics')
 ax.xaxis.set_major_locator(MaxNLocator(integer=True))
sns.despine()
plt.show() 
<categorify>
submission_df = prepare_submission(submission_df, test_df )
class Preprocessor:
def __init__(self, embeddings: dict):
self.embeddings = embeddings
def build_tf_dict(self, sentences: list):

 tf_dict = defaultdict(int)
 for sentence in sentences:
for word in sentence:
tf_dict[word] += 1
return tf_dict
def check_coverage(self, tf_dictionary: dict):

 in_vocabulary = defaultdict(int)
 out_of_vocabulary = defaultdict(int)
 in_count = 0
out_count = 0
for word in tf_dictionary:
if word in self.embeddings:
in_vocabulary[word] = self.embeddings[word]
in_count += tf_dictionary[word]
else:
out_of_vocabulary[word] = tf_dictionary[word]
out_count += tf_dictionary[word]
percent_tf = len(in_vocabulary)/ len(tf_dictionary)
 percent_all = in_count /(in_count + out_count)
 print('Found embeddings for {:.2%} of vocabulary and {:.2%} of all text'.format(percent_tf, percent_all))
return sorted(out_of_vocabulary.items() , key=operator.itemgetter(1)) [::-1]
def clean_punctuation(self, text: list):
result = text
for punct in "/-'":
result = result.replace(punct, ' ')
 for punct in '&':
result = result.replace(punct, f' {punct} ')
 for punct in '?!.,"
result = result.replace(punct, '')
 return result
def clean_digits(self, text: list):
result = text
result = re.sub('[0-9]{5,}', '
result = re.sub('[0-9]{4}', '
result = re.sub('[0-9]{3}', '
result = re.sub('[0-9]{2}', '
return result
def clean_misspelling(self, text: list):
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def apply_cleaning_function(self, fn, texts: list, description = ""):
result = [fn(text)for text in texts]
sentences = [text.split() for text in result]
tf_dict = self.build_tf_dict(sentences)
 oov = self.check_coverage(tf_dict)
 return result
def preprocess_for_embeddings_coverage(self, texts: list):
result = texts
sentences = [text.split() for text in result]
tf_dict = self.build_tf_dict(sentences)
 oov = self.check_coverage(tf_dict)
 result = self.apply_cleaning_function(lambda x: self.clean_punctuation(x), result, "Cleaning punctuation...")
 return result<import_modules>
def adapt_percent_in_submission() :
previous_match = None
for i, r in submission_df.iterrows() :
in_training = train_df[(train_df['Patient'] == r['Patient'])&(train_df['Weeks'] == r['Weeks'])]
if(len(in_training)> 0):
previous_match = in_training['Percent'].item()
submission_df.iloc[i, submission_df.columns.get_loc('Percent')] = previous_match
elif previous_match is not None:
submission_df.iloc[i, submission_df.columns.get_loc('Percent')] = previous_match
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
train_df['WHERE'] = 'train'
test_df['WHERE'] = 'val'
submission_df['WHERE'] = 'test'
data = train_df.append([test_df, submission_df] )
def make_model(emb_weights):
tokenized_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name="tokenized_input")
 embedding = Embedding(MAX_WORDS, EMBEDDINGS_LOADED_DIMENSIONS,
weights=[emb_weights],
trainable=False )(tokenized_input)
 embedding = SpatialDropout1D(0.25 )(embedding)
 lstm = Bidirectional(CuDNNGRU(64, return_sequences=True))(embedding)
 lstm = SpatialDropout1D(0.25 )(lstm)
 lstm = Bidirectional(CuDNNGRU(32, return_sequences=True))(lstm)
 a = Attention(MAX_SEQUENCE_LENGTH )(lstm)
 d1 = Dense(32 )(a)
 d1 = Dropout(0.25 )(d1)
 d1 = BatchNormalization()(d1)
 out = Dense(1, activation='sigmoid' )(d1)
 model = Model(inputs=[tokenized_input], outputs=out)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<prepare_x_and_y>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
X = question_texts
Y = question_targets<split>
base = data.loc[data.Weeks == data.min_week]
train_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size=0.01)
 test_predictions = []
kaggle_predictions = []
embedding_files = [
".. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin",
".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt",
".. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec",
".. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt"]
load_embedding_functions = [
lambda: load_embeddings(embedding_files[1])]
for index, load_embeddings_fn in enumerate(load_embedding_functions):
x_train = np.array(train_X)
 y_train = np.array(train_Y)
 pretrained_embeddings = load_embeddings_fn()
preprocessor = Preprocessor(pretrained_embeddings)
 x_train = preprocessor.preprocess_for_embeddings_coverage(x_train)
 x_kaggle = preprocessor.preprocess_for_embeddings_coverage(kaggle_texts)
 x_test = preprocessor.preprocess_for_embeddings_coverage(test_X)
 tokenizer = Tokenizer(num_words=MAX_WORDS)
 tokenizer.fit_on_texts(x_train)
 x_train = pad_sequences(tokenizer.texts_to_sequences(x_train), maxlen=MAX_SEQUENCE_LENGTH)
 x_kaggle = pad_sequences(tokenizer.texts_to_sequences(x_kaggle), maxlen=MAX_SEQUENCE_LENGTH)
 x_test = pad_sequences(tokenizer.texts_to_sequences(x_test), maxlen=MAX_SEQUENCE_LENGTH)
 pretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS)
 model = make_model(pretrained_emb_weights)
 splits = list(StratifiedKFold(n_splits=10, shuffle=True, random_state=317 ).split(x_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
x = np.array(x_train[train_idx])
 y = np.array(y_train[train_idx])
 X_val = np.array(x_train[valid_idx])
 y_val = np.array(y_train[valid_idx])
 epoch_callback = EpochMetricsCallback()
history = model.fit(x=x, y=y, validation_data=(X_val, y_val),
batch_size=BATCH_SIZE, epochs=2, verbose=2,
callbacks=[epoch_callback])
 display_model_history(history)
 display_model_epoch_metrics(epoch_callback)
 kaggle_predictions.append(model.predict([x_kaggle], batch_size=1024, verbose=2))
test_predictions.append(model.predict([x_test], batch_size=1024, verbose=2))<save_to_csv>
base = base[['Patient','FVC', 'Percent']].copy()
base.columns = ['Patient','min_FVC', 'min_Percent']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base
avg = np.average(kaggle_predictions, axis=0)
 df_out = pd.DataFrame({"qid":df_test["qid"].values})
 df_out['prediction'] =(avg > THRESHOLD ).astype(int)
 df_out.to_csv("submission.csv", index=False )<compute_test_metric>
data = data.merge(base, on='Patient', how='left' )
avg = np.average(test_predictions, axis=0)
 f1s = []
precisions = []
recalls = []
Ts = [x * 0.01 for x in range(25, 55)]
for t in Ts:
pred =(avg > t ).astype(int)
 f1s.append(metrics.f1_score(test_Y, pred))
precisions.append(metrics.precision_score(test_Y, pred))
recalls.append(metrics.recall_score(test_Y, pred))
data = pd.DataFrame(data = {'F1': f1s,
'Precision': precisions,
'Recall': recalls},
index=Ts)
 ax = sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)
 ax.set(xlabel='Threshold', ylabel='Value', title='Threshold levels')
 sns.despine()
plt.show()
thresh = Ts[np.argmax(f1s)]
pred =(avg > thresh ).astype(int)
 f1 = metrics.f1_score(test_Y, pred)
 print("Test F1 {0:.4f} at threshold {1:.3f}".format(f1, thresh))
thresh = THRESHOLD
pred =(avg > thresh ).astype(int)
 f1 = metrics.f1_score(test_Y, pred)
 print("Test F1 {0:.4f} at threshold {1:.3f}".format(f1, thresh))<save_to_csv>
data['base_week'] = data['Weeks'] - data['min_week']
data['base_week'] = data['base_week']
del base
thresh = Ts[np.argmax(f1s)]
avg = np.average(kaggle_predictions, axis=0)
 df_out = pd.DataFrame({"qid":df_test["qid"].values})
 df_out['prediction'] =(avg > thresh ).astype(int)
 df_out.to_csv("submission.csv", index=False )<import_modules>
data[data['Patient'] == 'ID00421637202311550012437']
stop_words = set(stopwords.words('english')) 
<categorify>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
def clean_text(x):
x = str(x)
 for p in string.punctuation + '“…':
x = x.replace(p, ' ' + p + ' ')
 x = x.replace('_', '')
 x = re.sub("`","'", x)
 x = re.sub("(?i)n't",' not', x)
 x = re.sub("(?i)'re",' are', x)
 x = re.sub("(?i)'s",' is', x)
 x = re.sub("(?i)'d",' would', x)
 x = re.sub("(?i)'ll",' will', x)
 x = re.sub("(?i)'t",' not', x)
 x = re.sub("(?i)'ve",' have', x)
 x = re.sub("(?i)'m",' am', x)
 x = re.sub("(?i)n\’t",' not', x)
 x = re.sub("(?i)\’re",' are', x)
 x = re.sub("(?i)\’s",' is', x)
 x = re.sub("(?i)\’d",' would', x)
 x = re.sub("(?i)\’ll",' will', x)
 x = re.sub("(?i)\’t",' not', x)
 x = re.sub("(?i)\’ve",' have', x)
 x = re.sub("(?i)\’m",' am', x)
 x = re.sub('(?i)Quorans','Quora', x)
 x = re.sub('(?i)Qoura','Quora', x)
 x = re.sub('(?i)Quoran','Quora', x)
 x = re.sub('(?i)dropshipping','drop shipping', x)
 x = re.sub('(?i)HackerRank','Hacker Rank', x)
 x = re.sub('(?i)Unacademy','un academy', x)
 x = re.sub('(?i)eLitmus','India hire employees', x)
 x = re.sub('(?i)WooCommerce','Commerce', x)
 x = re.sub('(?i)hairfall','hair fall', x)
 x = re.sub('(?i)marksheet','mark sheet', x)
 x = re.sub('(?i)articleship','article ship', x)
 x = re.sub('(?i)cryptocurrencies','cryptocurrency', x)
 x = re.sub('(?i)coinbase','cryptocurrency', x)
 x = re.sub('(?i)altcoin','bitcoin', x)
 x = re.sub('(?i)altcoins','bitcoins', x)
 x = re.sub('(?i)litecoin','bitcoin', x)
 x = re.sub('(?i)litecoins','bitcoins', x)
 x = re.sub('(?i)demonetisation','demonetization', x)
 x = re.sub('(?i)ethereum','bitcoin', x)
 x = re.sub('(?i)ethereums','bitcoins', x)
 x = re.sub('(?i)quorans','quora', x)
 x = re.sub('(?i)Brexit','britan exit', x)
 x = re.sub('(?i)upwork','freelance', x)
 x = re.sub('(?i)Unacademy','un academy', x)
 x = re.sub('(?i)Blockchain','blockchain', x)
 x = re.sub('(?i)GDPR','General Data Protection Regulation', x)
 x = re.sub('(?i)Qoura','quora', x)
 x = re.sub('(?i)HackerRank','Hacker Rank', x)
 x = re.sub('(?i)Cryptocurrency','cryptocurrency', x)
 x = re.sub('(?i)Binance','cryptocurrency', x)
 x = re.sub('(?i)Redmi','mobile phone', x)
 x = re.sub('(?i)TensorFlow','Tensor Flow', x)
 x = re.sub('(?i)Golang','programming language', x)
 x = re.sub('(?i)eLitmus','India hire employees', x)
 return x<load_from_csv>
data = data.rename(columns={"Age": "age", "min_FVC": "BASE", "base_week": "week", "Percent": "percent"})
 FE += ['age','week','BASE', 'percent']
FE
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 print('Train shape : ', train_df.shape)
 print('Test shape : ', test_df.shape )<split>
train_df = data.loc[data.WHERE=='train']
test_df = data.loc[data.WHERE=='val']
submission_df = data.loc[data.WHERE=='test']
del data
train_df, val_df = train_test_split(train_df, test_size=0.2,random_state=123 )<feature_engineering>
train_df.sort_values(['Patient', 'Weeks'], inplace=True )
tqdm_notebook.pandas(desc="Example Desc")
 train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))
val_df['question_text'] = val_df['question_text'].progress_apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))<categorify>
y = train_df['FVC']
y
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x<feature_engineering>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )
train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))
val_df['question_text'] = val_df['question_text'].progress_apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))<categorify>
seed_everything(42 )
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x<feature_engineering>
C1_val = 70
C2_val = 1000
C1, C2 = C1_val, C2_val
q = np.array([0.2, 0.50, 0.8])
 def score(y_true, y_pred):
sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = np.max(sigma)
 delta = np.abs(y_true[:, 0] - fvc_pred)
 delta = np.min(delta)
 sq2 = np.sqrt(2.)
metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 def qloss(y_true, y_pred):
print('y_true.shape', y_true.shape)
 print('y_pred.shape', y_pred.shape)
 e = y_true - y_pred
v = np.max(q*e,(q-1)*e)
 return np.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
y_true = y_true.reshape(-1, 1)
 return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))
val_df['question_text'] = val_df['question_text'].progress_apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))<string_transform>
n_estimators = 30000
if not 'sub_row' in locals() :
sub_row = 0.75
if not 'bagging_freq' in locals() :
bagging_freq = 1
if not 'learning_rate' in locals() :
learning_rate = 0.4
leave_size = 4
lgb_params = {
"objective": 'quantile',
'n_jobs': 1,
'max_depth': leave_size + 1,
'num_leaves': 2**leave_size-1,
"min_data_in_leaf": 2**(leave_size + 1)-1,
"n_estimators": n_estimators,
'learning_rate': 8e-3,
'colsample_bytree': 0.9,
'boosting_type': 'gbdt',
"early_stopping_rounds": 100,
'verbosity': 1000,
"metric": ["rmse", "mse"]
}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<feature_engineering>
cat_feats = ['Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']
train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: replace_typical_misspell(x))
val_df['question_text'] = val_df['question_text'].progress_apply(lambda x: replace_typical_misspell(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: replace_typical_misspell(x))<define_variables>
NFOLD = 5
kf = KFold(n_splits=NFOLD, shuffle=False)
 pred = {a: np.zeros(( train_df.shape[0])) for a in q.tolist() }
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
train_df['question_text'] = train_df['question_text'].str.lower()
val_df['question_text'] = val_df['question_text'].str.lower()
test_df['question_text'] = test_df['question_text'].str.lower()
train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))
val_df['question_text'] = val_df['question_text'].progress_apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))<define_variables>
ensemble_weights = [6./7, 1./7]
assert np.sum(ensemble_weights)== 1.0
full_text = list(train_df['question_text'].values)+ list(val_df['question_text'].values)+list(test_df['question_text'].values )<import_modules>
def ols_quantile(m, X, q):
mean_pred = m.predict(X)
 se = np.sqrt(m.scale)
 return mean_pred + norm.ppf(q)* se
import pandas as pd
import numpy as np
from sklearn.preprocessing import OneHotEncoder
import keras
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D, GRU
from keras.layers import Bidirectional, GlobalMaxPool1D, MaxPooling1D, Add, Flatten
from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D, BatchNormalization
from keras.models import Model, load_model
from keras import initializers, regularizers, constraints, optimizers, layers, callbacks
from keras import backend as K
from keras.engine import InputSpec, Layer
from keras.optimizers import Adam, RMSprop
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint, LearningRateScheduler<train_model>
models = []
linear_models = []
for cnt,(tr_idx, val_idx)in tqdm(enumerate(kf.split(X)) , total=NFOLD):
X_train, y_train = X.loc[tr_idx], y.loc[tr_idx]
X_valid, y_valid = X.loc[val_idx], y.loc[val_idx]
print(f"FOLD {cnt}", X_train.shape, y_train.shape, X_valid.shape, y_valid.shape)
 for qi, quantile_alpha in enumerate(q.tolist()):
lgb_params['alpha'] = quantile_alpha
m_lgb_regressor = lgb.LGBMRegressor(**lgb_params)
 m_lgb_regressor.fit(X=X_train, y=y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)],
eval_names=['train mloss', 'valid mloss'],
eval_metric=lgb_params['metric'],
verbose=lgb_params['verbosity'],
early_stopping_rounds=lgb_params["early_stopping_rounds"],
categorical_feature=cat_feats)
 lin_model = sm.OLS(y_train, X_train ).fit()
lin_predict = ols_quantile(lin_model, X_valid, quantile_alpha)
 lgb_predict = m_lgb_regressor.predict(X_valid)
 pred[quantile_alpha][val_idx] = np.average([lin_predict, lgb_predict], axis = 0, weights=ensemble_weights)
 models.append(m_lgb_regressor)
 linear_models.append(lin_model )
tokenizer = Tokenizer(lower = True, filters = '')
 tokenizer.fit_on_texts(full_text )<string_transform>
full_preds = np.vstack([pred[a] for a in q] ).T
score(np.array(y ).reshape(-1, 1), full_preds )
train_tokenized = tokenizer.texts_to_sequences(train_df['question_text'])
 val_tokenized = tokenizer.texts_to_sequences(val_df['question_text'])
 test_tokenized = tokenizer.texts_to_sequences(test_df['question_text'] )<string_transform>
pred = []
for i in range(NFOLD):
cur_pred = []
for j, quantile in enumerate(q):
model_idx = i * 3 + j
model = models[model_idx]
lin_predict = ols_quantile(linear_models[model_idx], submission_df[FE], quantile)
 dbmc_predict = model.predict(submission_df[FE])
 cur_pred.append(np.average([lin_predict, dbmc_predict], axis = 0, weights=ensemble_weights))
pred.append(np.array(cur_pred ).T )
max_len = 70
X_train = pad_sequences(train_tokenized, maxlen = max_len)
 X_val = pad_sequences(val_tokenized, maxlen = max_len)
 X_test = pad_sequences(test_tokenized, maxlen = max_len )<define_variables>
submission_df['FVC1'] = final_preds[:,1]
submission_df['Confidence1'] = final_preds[:, 2] - final_preds[:, 0]
embedding_path = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"<define_variables>
submission_df.loc[~submission_df.FVC1.isnull() ,'FVC'] = submission_df.loc[~submission_df.FVC1.isnull() ,'FVC1']
submission_df.loc[~submission_df.FVC1.isnull() ,'Confidence'] = submission_df.loc[~submission_df.FVC1.isnull() ,'Confidence1']
embed_size = 300
max_features = 100000<compute_test_metric>
submission_df[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )
<categorify><EOS> 
submission_final_df = pd.read_csv("submission.csv" )